From 4af9483219f799443fb6dbde2ba089b295a8bfb7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 10 Sep 2026 22:40:02 +0200 Subject: [PATCH 1/9] declaration: an image may be named by the digest of its own configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A manifest digest is assigned by a registry on push, so insisting on one meant a registry had to exist before the thing that lets a mesh have a registry could start — a dependency the pinning rule created by accident, not a pin. The mesh's own control plane is built from source and lives in no public registry. A bare sha256:... names an image the machine already holds, by the digest of its own configuration: immutable and unforgeable in exactly the way the rule asks for. Absent, it says so plainly rather than failing at a pull nothing serves. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/apply/apply.go | 9 ++++++++ internal/declaration/declaration.go | 27 +++++++++++++++++++++--- internal/declaration/declaration_test.go | 26 +++++++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6b2c65d..baba8d4 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1154,6 +1154,15 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error { if _, err := run(ctx, cri, "image", "inspect", image); err == nil { return nil } + // An image named by the digest of its own configuration is one this machine was supposed to + // already hold — built here, or handed over. There is no registry that answers for it, so + // pulling would fail somewhere that names a network problem instead of a missing image. + if strings.HasPrefix(image, "sha256:") { + return fmt.Errorf( + "image %s is not on this machine, and an image named by its own digest cannot be "+ + "fetched: nothing serves it. Build it here, or load it, before applying this", + image) + } if _, err := run(ctx, cri, "pull", image); err != nil { return fmt.Errorf("pulling image %s: %w", image, err) } diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 3ffa8cf..84a6824 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -874,20 +874,41 @@ func checkMode(where, mode string) []string { return nil } -// checkImage insists on a digest. +// checkImage insists on content, not on a name. // // A tag moves and a digest does not. The bundle's whole claim is that what it names is exact // (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever // that tag points at on the day the host happens to run. +// +// **Two forms say something exact, and only one of them needs a registry.** `name@sha256:…` is a +// manifest digest, which a registry assigns on push. A bare `sha256:…` is an image the machine +// already holds, addressed by the digest of its own configuration — equally immutable, equally +// unforgeable, and requiring nothing to have served it. +// +// That second form is what a first machine needs. The mesh's own control plane exists in no public +// registry and never will: it is built from source, and until this mesh has a registry of its own +// there is nowhere to push it to and therefore no manifest digest to name it by. Insisting on one +// would mean a registry has to exist before the thing that lets a mesh have a registry can start — +// which is not a pin, it is a dependency the rule accidentally created. A machine that built an +// image, or was handed one, can name it by what it is. func checkImage(where, image string) []string { if image == "" { return []string{where + ": a container needs an image"} } + // An image this machine holds, named by the digest of its own configuration. + if strings.HasPrefix(image, "sha256:") { + if len(image) != len("sha256:")+64 { + return []string{fmt.Sprintf( + "%s: image id %q is not a sha256 digest", where, image)} + } + return nil + } name, digest, found := strings.Cut(image, "@") if !found || name == "" { return []string{fmt.Sprintf( - "%s: image %q is not pinned. Write it as name@sha256:... — a tag moves, and a "+ - "bundle that pinned a tag would not be pinned", where, image)} + "%s: image %q is not pinned. Write it as name@sha256:… — or as sha256:… for an image "+ + "this machine already holds. A tag moves, and a bundle that pinned a tag would "+ + "not be pinned", where, image)} } if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { return []string{fmt.Sprintf( diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index 46734b3..11244bc 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -219,6 +219,32 @@ func TestAnImageMustBePinnedByDigest(t *testing.T) { } } +// An image the machine already holds, addressed by the digest of its own configuration. +// +// **The form a first machine needs.** A manifest digest is assigned by a registry on push, so +// insisting on one means a registry has to exist before the thing that lets a mesh have a registry +// can start. The mesh's own control plane is built from source and lives in no public registry; a +// machine that built it, or was handed it, names it by what it is — a content address, not a name, +// and immutable in exactly the way the rule asks for. +func TestAnImageTheMachineHoldsIsNamedByItsOwnDigest(t *testing.T) { + held := "sha256:" + strings.Repeat("b", 64) + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-control","image":"` + held + `"} + ]}`)); err != nil { + t.Errorf("an image named by its own digest was refused: %v", err) + } + + // Still a digest, though. A truncated one names several images, and which one ran would be + // whichever the runtime happened to match first. + for _, bad := range []string{"sha256:abc", "sha256:", "sha256:" + strings.Repeat("b", 63)} { + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-control","image":"` + bad + `"} + ]}`)); err == nil { + t.Errorf("image id %q was accepted and is not a digest", bad) + } + } +} + func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { // The field-set check must cover the types added last, not only the three it was written // for. A package that carries a `content` is a control plane believing it asked for -- 2.54.0 From b82ab95f74ae945fca2be3bce0efd1a04f369cf0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 10 Sep 2026 23:17:30 +0200 Subject: [PATCH 2/9] mesh-bootstrap: the first-node procedure, as a program rather than a test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only complete written-down copy of how a mesh is stood up was an integration test in the lab. That is why every bootstrap gap kept being found late: an install procedure that lives as a test fixture is exercised by whoever writes tests, never by whoever installs. This is that procedure. A separate binary, not a mesh-host subcommand. mesh-host says of itself that it connects to nothing and listens on nothing and that what it applies comes from a file, and that sentence is what makes an always-running root daemon auditable. An installer loads images and interrogates a control plane. Same tier, different program. The control plane's image is carried, not built and not fetched. The forge that holds its source runs on the mesh, so a bootstrap that had to fetch it would need a mesh in order to raise one. Embedding breaks that cycle the way the carried bundle breaks "copy it onto a machine and run it". The image id is read out of the saved tar before the runtime is asked anything, which is what makes the load idempotent: the installer can ask whether the machine already holds exactly this. Five steps, each idempotent and each saying whether it found or changed something, because this is run over and over by somebody getting a machine working. It stops at a running substrate with a control plane that replies — enrolment, the module catalogue and assignment are the next stage and are deliberately absent. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- .gitignore | 4 + Makefile | 26 ++- cmd/mesh-bootstrap/main.go | 192 ++++++++++++++++++++ cmd/mesh-bootstrap/main_test.go | 113 ++++++++++++ internal/bootstrap/apply.go | 134 ++++++++++++++ internal/bootstrap/apply_test.go | 91 ++++++++++ internal/bootstrap/bootstrap.go | 260 +++++++++++++++++++++++++++ internal/bootstrap/load.go | 122 +++++++++++++ internal/bootstrap/load_test.go | 211 ++++++++++++++++++++++ internal/bootstrap/preflight.go | 211 ++++++++++++++++++++++ internal/bootstrap/preflight_test.go | 126 +++++++++++++ internal/bootstrap/rewrite.go | 241 +++++++++++++++++++++++++ internal/bootstrap/rewrite_test.go | 223 +++++++++++++++++++++++ internal/bootstrap/verify.go | 167 +++++++++++++++++ internal/bootstrap/verify_test.go | 168 +++++++++++++++++ internal/image/control-plane.tar | 8 + internal/image/image.go | 173 ++++++++++++++++++ internal/image/image_test.go | 152 ++++++++++++++++ 18 files changed, 2620 insertions(+), 2 deletions(-) create mode 100644 cmd/mesh-bootstrap/main.go create mode 100644 cmd/mesh-bootstrap/main_test.go create mode 100644 internal/bootstrap/apply.go create mode 100644 internal/bootstrap/apply_test.go create mode 100644 internal/bootstrap/bootstrap.go create mode 100644 internal/bootstrap/load.go create mode 100644 internal/bootstrap/load_test.go create mode 100644 internal/bootstrap/preflight.go create mode 100644 internal/bootstrap/preflight_test.go create mode 100644 internal/bootstrap/rewrite.go create mode 100644 internal/bootstrap/rewrite_test.go create mode 100644 internal/bootstrap/verify.go create mode 100644 internal/bootstrap/verify_test.go create mode 100644 internal/image/control-plane.tar create mode 100644 internal/image/image.go create mode 100644 internal/image/image_test.go diff --git a/.gitignore b/.gitignore index 235d26c..d9cc9a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /mesh-host +/mesh-bootstrap /dist/ +# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an +# interrupted release build cannot commit a twenty-megabyte tar by accident. +/internal/image/control-plane.tar.placeholder diff --git a/Makefile b/Makefile index ebe96e7..d5bf716 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LDFLAGS := -s -w -X main.builtFor=$(SYSTEM) -X main.version=$(VERSION) # a second file to arrive with it is not "copy it and run it". BUNDLE ?= -.PHONY: check test vet fmt build clean host +.PHONY: check test vet fmt build clean host hosts bootstrap packaging-test check: fmt vet test packaging-test build @@ -57,5 +57,27 @@ host: exit $$status @echo "built for $(SYSTEM) carrying $(BUNDLE)" +# The installer, carrying the control plane's image: +# make bootstrap IMAGE=mesh-control:v1.2.3 +# +# The image is BUILT ELSEWHERE and handed over — mesh-control's own `make image` — and embedded +# here at release time. Not built on the machine being bootstrapped, and not fetched: the forge +# that holds mesh-control's source runs on the mesh, so a bootstrap that had to fetch or build the +# control plane would need a mesh in order to raise one. Carrying it breaks that cycle, the same +# way carrying the bundle breaks the "copy it onto a machine and run it" one (novox/hq ADR 0005). +# +# The saved image occupies the embed slot for the length of one build and the placeholder goes +# back, exactly as `host:` does with the bundle. Nothing large is ever committed. +bootstrap: + @test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; } + @docker image inspect "$(IMAGE)" >/dev/null 2>&1 || { echo "this machine does not hold $(IMAGE) — build it in mesh-control with 'make image'"; exit 1; } + @cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder + @docker save --output internal/image/control-plane.tar "$(IMAGE)" + @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o mesh-bootstrap ./cmd/mesh-bootstrap; \ + status=$$?; \ + mv internal/image/control-plane.tar.placeholder internal/image/control-plane.tar; \ + exit $$status + @echo "built mesh-bootstrap carrying $(IMAGE)" + clean: - rm -f mesh-host + rm -f mesh-host mesh-bootstrap diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go new file mode 100644 index 0000000..894462c --- /dev/null +++ b/cmd/mesh-bootstrap/main.go @@ -0,0 +1,192 @@ +// Command mesh-bootstrap brings a mesh into existence on a bare machine. +// +// Tier 0, beside `mesh-host` and not inside it. Bootstrapping is done by hand and it changes a +// machine, which is what tier 0 is (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) — but +// `mesh-host` states of itself that it connects to nothing and listens on nothing and that what it +// applies comes from a file, and that is the whole reason an always-running root daemon can be +// audited by reading one page. An installer that loads images and interrogates a control plane +// cannot be folded into it without making that sentence false. Same tier, same repository, +// different program. +// +// What it does not do is enrol this machine, register modules or assign them. It stops at a +// running substrate with a control plane that answers, which is a mesh of one node. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net" + "os" + "os/signal" + "syscall" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// version is stamped at build time. Unset in a development build, and said so rather than +// defaulted to something that looks like a release. +var version = "development build" + +const ( + defaultTemplate = "substrate.lock" + defaultOut = "/var/lib/mesh-host/substrate.lock" +) + +const usage = `mesh-bootstrap — make a bare machine into a mesh + + bootstrap preflight, load, bundle, apply, verify (the default) + version + + --bundle the substrate template to build this machine's bundle from + (default ` + defaultTemplate + `) + --out where the produced bundle is written, for a person to read + (default ` + defaultOut + `) + --state where this node records what it has applied + (default ` + store.DefaultPath + `) + --system which operating system this is; by default it is asked + --timeout how long any single probe may take (default 30s) + --wait how long a thing that is merely starting is given (default 3m) + --dry-run everything that does not change the machine + --json machine-readable output + +It carries the control plane's image and applies a substrate. Every step is idempotent: +run it again after fixing whatever it named, and the steps that already succeeded say so. +` + +func main() { + // Ctrl-C must stop the installer, not be swallowed by whatever it is waiting for — and it + // waits on pulls, on a runtime starting, and on a control plane opening its stores. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + command, opts, jsonOut, err := parseArgs(os.Args[1:]) + if err == nil { + err = run(ctx, command, opts, jsonOut) + } + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-bootstrap: %v\n", err) + os.Exit(1) + } +} + +// parseArgs takes an optional subcommand first, then its flags. +// +// Parsed in a loop for the reason `mesh-host` records: the standard library stops at the FIRST +// non-flag argument, so a flag sitting after one is silently dropped and the command exits zero +// having ignored what it was asked. That fault has been paid for twice in this repository and is +// not being paid for a third time. +func parseArgs(args []string) (string, bootstrap.Options, bool, error) { + opts := bootstrap.Options{ + Template: defaultTemplate, + Out: defaultOut, + State: store.DefaultPath, + // Longer than the host's 10s: these probes reach a container runtime that may be busy + // pulling, and a probe that times out on a working machine is a false refusal. + Timeout: 30 * time.Second, + // A socket-activated runtime queued behind the network, and a control plane running its + // first `initdb`-shaped wait, are both minutes rather than seconds. + Wait: 3 * time.Minute, + } + var jsonOut bool + + command := "bootstrap" + if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' { + command = args[0] + args = args[1:] + } + + set := newFlagSet(&opts, &jsonOut) + var positionals []string + rest := args + for { + if err := set.Parse(rest); err != nil { + return "", opts, false, err + } + rest = set.Args() + if len(rest) == 0 { + break + } + positionals = append(positionals, rest[0]) + rest = rest[1:] + } + + // Refused rather than ignored: a mistyped argument that changes nothing and reports success is + // worse than an error, and this program's whole job is to change a machine. + if len(positionals) > 0 { + return "", opts, false, fmt.Errorf( + "unexpected argument %q — try `mesh-bootstrap help`", positionals[0]) + } + return command, opts, jsonOut, nil +} + +func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { + set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError) + set.SetOutput(os.Stderr) + set.Usage = func() { fmt.Fprint(os.Stderr, usage) } + set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from") + set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written") + set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied") + set.StringVar(&opts.System, "system", opts.System, "which operating system this is") + set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take") + set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given") + set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine") + set.BoolVar(jsonOut, "json", false, "machine-readable output") + return set +} + +func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error { + switch command { + case "bootstrap": + say := func(line string) { + if !jsonOut { + fmt.Println(line) + } + } + result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{ + Run: apply.ExecRunner, + Dial: dial, + }, say) + + // Printed whichever way it went. What the installer got through before it stopped is on + // the machine either way, and a report that only exists on success describes a machine + // nobody has (novox/hq ADR 0018). + if jsonOut { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if encodeErr := encoder.Encode(result); encodeErr != nil && err == nil { + return encodeErr + } + } + return err + + case "version": + fmt.Println(version) + return nil + + case "help", "-h", "--help": + fmt.Fprint(os.Stderr, usage) + return nil + + default: + return fmt.Errorf("unknown command %q — try `mesh-bootstrap help`", command) + } +} + +// dial answers whether a TCP address responds. +// +// A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a +// connection to exactly this address. A machine whose DNS resolves and whose route is missing +// passes a lookup and fails the thing that matters. +func dial(ctx context.Context, address string) error { + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return err + } + return conn.Close() +} diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go new file mode 100644 index 0000000..6731428 --- /dev/null +++ b/cmd/mesh-bootstrap/main_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "flag" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// Argument handling gets tests for the reason `mesh-host` records: the standard library stops +// parsing at the first non-flag argument, so a flag sitting after one is silently dropped and the +// command exits zero having ignored what it was asked. Here that would mean `--dry-run` ignored on +// a program whose whole job is to change a machine. + +func TestBootstrapIsWhatItDoesWithNoCommand(t *testing.T) { + // Running the installer with nothing but flags must install, not print usage: the command is + // the reason the binary exists, and making somebody type its name twice buys nothing. + command, opts, _, err := parseArgs([]string{"--dry-run"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if command != "bootstrap" { + t.Errorf("command = %q, want bootstrap", command) + } + if !opts.DryRun { + t.Error("--dry-run before any subcommand was ignored") + } +} + +func TestFlagsAreReadWhereverTheySit(t *testing.T) { + for _, args := range [][]string{ + {"bootstrap", "--dry-run", "--bundle", "s.lock", "--json"}, + {"bootstrap", "--json", "--bundle=s.lock", "--dry-run"}, + {"--bundle", "s.lock", "--dry-run", "--json"}, + } { + _, opts, jsonOut, err := parseArgs(args) + if err != nil { + t.Errorf("%v: unexpected error: %v", args, err) + continue + } + if !opts.DryRun || !jsonOut || opts.Template != "s.lock" { + t.Errorf("%v parsed as dry-run=%v json=%v bundle=%q", + args, opts.DryRun, jsonOut, opts.Template) + } + } +} + +func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) { + // Asymmetric cost: an error is a moment's annoyance, and a silently dropped --dry-run is a + // machine changed by somebody who asked for it not to be. + if _, _, _, err := parseArgs([]string{"bootstrap", "--dry-runn"}); err == nil { + t.Fatal("a mistyped flag was accepted") + } +} + +func TestAnUnexpectedArgumentIsRefused(t *testing.T) { + if _, _, _, err := parseArgs([]string{"bootstrap", "substrate.lock"}); err == nil { + t.Fatal("a stray argument was ignored rather than refused — the bundle is --bundle") + } +} + +func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) { + // The usage text is a promise. A default that drifts from what is printed is a small lie that + // costs somebody an afternoon in front of a machine that will not come up. + _, opts, jsonOut, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Template != defaultTemplate { + t.Errorf("default bundle is %q; the usage text says %q", opts.Template, defaultTemplate) + } + if opts.Out != defaultOut { + t.Errorf("default out is %q; the usage text says %q", opts.Out, defaultOut) + } + if opts.State != store.DefaultPath { + t.Errorf("default state is %q; the host's own default is %q", opts.State, store.DefaultPath) + } + if opts.Timeout != 30*time.Second { + t.Errorf("default timeout is %s; the usage text says 30s", opts.Timeout) + } + if opts.Wait != 3*time.Minute { + t.Errorf("default wait is %s; the usage text says 3m", opts.Wait) + } + if opts.DryRun || jsonOut || opts.System != "" { + t.Error("something is on by default that the usage text describes as a flag") + } +} + +// Every flag the usage text promises must exist, and every flag that exists must be in the usage +// text. The two drifting apart is how a program acquires a feature nobody can find and a +// documented option that does nothing. +func TestTheUsageTextAndTheFlagsAgree(t *testing.T) { + var opts bootstrap.Options + var jsonOut bool + set := newFlagSet(&opts, &jsonOut) + + declared := map[string]bool{} + set.VisitAll(func(f *flag.Flag) { declared[f.Name] = true }) + + for name := range declared { + if !strings.Contains(usage, "--"+name) { + t.Errorf("--%s exists and the usage text does not mention it", name) + } + } + for _, promised := range []string{"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json"} { + if !declared[promised] { + t.Errorf("the usage text promises --%s and no such flag exists", promised) + } + } +} diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go new file mode 100644 index 0000000..e792059 --- /dev/null +++ b/internal/bootstrap/apply.go @@ -0,0 +1,134 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Runner is the same runner every applier in this repository takes. +type Runner = apply.Runner + +// ApplyBundle raises the substrate, through the host's own apply. +// +// **This calls `internal/apply` rather than running the `mesh-host` binary**, and that is worth +// stating because shelling out would have been easier. The installer and the host must apply a +// declaration identically — same removal pass, same read-backs, same refusal model, same record of +// what this machine now owns — and two code paths that must behave the same are two code paths +// that will not. The `mesh-host` binary is also not guaranteed to be on a machine this program is +// raising, which would make the installer depend on the thing it installs. +// +// It applies under `store.OriginCarried`, which is the same origin `mesh-host reconcile` uses and +// is not a detail: what the substrate raised must be invisible to the removal pass of a +// declaration that later arrives from the control plane, or the first thing the mesh tells this +// node would tear down the mesh (novox/hq 04-ISSUES/010). +// +// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version, +// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is +// not one. +func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration, + source string, run Runner, say func(string)) (apply.Report, error) { + + // Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host` + // does: finding out half way through is the half-configured machine tier 0 exists to prevent. + if err := system.Check(sys, d); err != nil { + return apply.Report{}, err + } + + known, err := store.Load(o.State) + if err != nil { + return apply.Report{}, err + } + + report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + + // Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a + // failure is on the machine either way, and a host that did not record it would believe it + // owns less than it does and leave that behind for ever. + if saveErr := store.Save(o.State, updated); saveErr != nil { + if applyErr != nil { + return report, fmt.Errorf("%w\n\nand this node's state could not be saved: %v", + applyErr, saveErr) + } + return report, saveErr + } + if applyErr != nil { + return report, fmt.Errorf("%w\n\nThe machine is in whatever state that left it. Fix what "+ + "is named above and run this again — every step is idempotent, and the ones that "+ + "already succeeded will say so", applyErr) + } + return report, nil +} + +// refuseSealed is what happens when a bundle contains a file the mesh sealed to this node. +// +// It cannot happen and it is refused with a sentence rather than a nil dereference. A sealing key +// is generated at enrolment (`internal/identity`), and enrolment is something that happens on a +// mesh — which is the thing this program is raising. A substrate bundle carrying a sealed file +// would be a bundle written for a node that has already joined. +func refuseSealed(string) ([]byte, error) { + return nil, errors.New( + "this bundle contains a file sealed to a node's key, and a machine that has not enrolled " + + "has no such key. A substrate is applied before any mesh exists, so it can carry no " + + "secret the mesh sealed") +} + +// WorkOutSystem decides which half of the host applies things on this machine, and proves it. +// +// `mesh-host` pins this at link time because it is built for one operating system and refuses to +// touch a machine without knowing which (novox/hq ADR 0005). An installer run by hand has no +// link-time to pin it at, so it asks — but it does not guess: every system already knows how to +// prove it is the one it claims to be, by asking its package database about a package that is +// certainly there. Exactly one may answer. +// +// A machine where none answers is refused with what each of them said, because "unsupported +// system" is a sentence nobody can act on and "pacman does not answer here" is. +func WorkOutSystem(ctx context.Context, run Runner, named string) (system.System, error) { + if strings.TrimSpace(named) != "" { + chosen, err := system.For(named) + if err != nil { + return nil, err + } + if err := chosen.Confirm(ctx, run); err != nil { + return nil, fmt.Errorf("--system %s was given, and this machine says otherwise: %w", + named, err) + } + return chosen, nil + } + + var answered []system.System + var refusals []string + for _, candidate := range system.All() { + if err := candidate.Confirm(ctx, run); err != nil { + refusals = append(refusals, fmt.Sprintf(" %s: %v", candidate.Name(), err)) + continue + } + answered = append(answered, candidate) + } + + switch len(answered) { + case 1: + return answered[0], nil + case 0: + return nil, fmt.Errorf( + "this machine is none of the systems this installer knows how to change, so nothing "+ + "was attempted:\n%s\nName one with --system if it is really one of them and its "+ + "package database is merely unwell", strings.Join(refusals, "\n")) + default: + var names []string + for _, s := range answered { + names = append(names, s.Name()) + } + return nil, fmt.Errorf( + "this machine answers as %s at once, and the installer must not choose between them: "+ + "package names and unit names differ, and picking wrong misconfigures the machine "+ + "quietly. Say which with --system", strings.Join(names, " and ")) + } +} diff --git a/internal/bootstrap/apply_test.go b/internal/bootstrap/apply_test.go new file mode 100644 index 0000000..7a3c8ea --- /dev/null +++ b/internal/bootstrap/apply_test.go @@ -0,0 +1,91 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" +) + +// `mesh-host` is built for one operating system and pins it at link time. An installer run by hand +// has no link time, so it asks — and it does not guess: each system already knows how to prove it +// is the one it claims to be, by asking its package database about a package that is certainly +// there. Getting this wrong installs with the wrong package manager and the wrong unit names. + +func TestTheMachineIsAskedWhichSystemItIs(t *testing.T) { + // Only pacman answers, so this is the arch host and nothing had to be told so. + onlyPacman := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "pacman" { + return "pacman 7.0.0-1\n", nil + } + return "", errors.New("command not found") + } + + chosen, err := WorkOutSystem(context.Background(), onlyPacman, "") + if err != nil { + t.Fatal(err) + } + if chosen.Name() != "arch" { + t.Errorf("this machine was worked out to be %q", chosen.Name()) + } +} + +// A machine that is none of them is refused with what each of them said. "Unsupported system" is +// a sentence nobody can act on; "pacman does not answer here" is. +func TestAMachineThatIsNoneOfThemIsRefusedWithWhatEachSaid(t *testing.T) { + nothing := func(context.Context, string, ...string) (string, error) { + return "", errors.New("command not found") + } + + _, err := WorkOutSystem(context.Background(), nothing, "") + if err == nil { + t.Fatal("a machine that answers as no known system was accepted") + } + for _, wanted := range []string{"arch:", "alpine:", "--system"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a machine that was TOLD what it is still has to prove it. Installing the arch half of the +// host on Alpine must say so once, at the start, rather than failing later inside pacman. +func TestASystemThatWasNamedIsStillProved(t *testing.T) { + onlyApk := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "apk" { + return "apk-tools-2.14.0\n", nil + } + return "", errors.New("command not found") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "arch"); err == nil { + t.Fatal("--system arch was believed on a machine where pacman does not answer") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "alpine"); err != nil { + t.Errorf("--system alpine was refused on a machine where apk answers: %v", err) + } +} + +func TestASystemNobodyHasBuiltIsRefusedByName(t *testing.T) { + anything := func(context.Context, string, ...string) (string, error) { return "", nil } + _, err := WorkOutSystem(context.Background(), anything, "debian") + if err == nil { + t.Fatal("--system debian was accepted, and no debian host is built") + } + if !strings.Contains(err.Error(), "arch") { + t.Errorf("the refusal does not say which systems exist: %v", err) + } +} + +// A substrate is applied before any mesh exists, so it can carry no secret the mesh sealed — there +// is no key to open one with. Refused with a sentence rather than a nil dereference. +func TestASealedFileInASubstrateIsRefusedWithAReason(t *testing.T) { + _, err := refuseSealed("anything") + if err == nil { + t.Fatal("a sealed file in a substrate bundle was accepted") + } + if !strings.Contains(err.Error(), "has not enrolled") { + t.Errorf("the refusal does not say why there is no key: %v", err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 0000000..428aefc --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,260 @@ +// Package bootstrap brings a mesh into existence on a bare machine. +// +// **Why this is a program at all.** Until now the only complete written-down copy of the +// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is +// why every gap in it kept being found late and by accident: an install procedure that lives as a +// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is +// that procedure, made into the thing it always was. +// +// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by +// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is +// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and +// listens on nothing and that what it applies comes from a file — a property that is what makes an +// always-running root daemon auditable, and that must stay literally true. This program pulls +// images and asks a running control plane questions. Same tier, same repository, different binary. +// +// **No registry is required for the mesh's own image, and no source either.** The control plane +// exists in no registry by design, and the forge that holds its source runs on the mesh — so a +// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer +// carries the image (`internal/image`) and names it by its image id: the sha256 of its own +// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq +// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream +// `name@sha256:` references and are pulled from the internet like anything else. +package bootstrap + +import ( + "context" + "fmt" + "time" +) + +// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence +// nobody can act on and this will be run over and over by somebody getting a machine working. +type Step string + +const ( + StepPreflight Step = "preflight" + StepLoad Step = "load" + StepBundle Step = "bundle" + StepApply Step = "apply" + StepVerify Step = "verify" +) + +// Steps in the order they happen, so a failure can say "step 2 of 5". +var Steps = []Step{StepPreflight, StepLoad, StepBundle, StepApply, StepVerify} + +// Error is a failure, named by the step it happened in. +type Error struct { + Step Step + Err error +} + +func (e *Error) Error() string { + at := 0 + for i, s := range Steps { + if s == e.Step { + at = i + 1 + } + } + return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err) +} + +func (e *Error) Unwrap() error { return e.Err } + +func failed(step Step, err error) error { + if err == nil { + return nil + } + return &Error{Step: step, Err: err} +} + +// Options are the things that differ between machines. +type Options struct { + // Template is the substrate bundle this machine's own bundle is made from. + Template string + // Out is where the produced bundle is written, so a person can read what was applied. + Out string + // State is where the host records what it has applied here — the same file `mesh-host` reads, + // because what this raises the host must afterwards own. + State string + // System is which half of the host applies things. Empty means ask the machine. + System string + // DryRun does everything that does not change the machine. + DryRun bool + // Timeout bounds any single probe. + Timeout time.Duration + // Wait is how long something that is merely starting is given: a socket-activated container + // runtime, a control plane opening its stores. + Wait time.Duration +} + +// Deps are the ways this program reaches outside itself. Injected so the whole of it can be +// tested without a container runtime, a network, or a machine to break — the same reason +// `internal/apply` takes a Runner (novox/hq ADR 0017). +type Deps struct { + // Run executes a command. apply.ExecRunner in production. + Run Runner + // Dial reports whether a TCP address answers, for "can this machine reach the registries the + // bundle names". + Dial func(ctx context.Context, address string) error +} + +// Result is what the bootstrap did, in the shape `--json` prints. +type Result struct { + System string `json:"system"` + DryRun bool `json:"dry-run,omitempty"` + + // Image is the control plane's image id — what the produced bundle names it by. + Image string `json:"image,omitempty"` + // ImageTags is what that image was called when it was saved. Decoration, for a person. + ImageTags []string `json:"image-tags,omitempty"` + // ImageHeld is true when the machine already held it and nothing was loaded. + ImageHeld bool `json:"image-already-held,omitempty"` + + // Bundle is where the produced bundle was written, and what was done to produce it. + Bundle string `json:"bundle,omitempty"` + BundleWas string `json:"bundle-replaced,omitempty"` + BundlePlaces int `json:"bundle-places,omitempty"` + BundleWrote bool `json:"bundle-written,omitempty"` + + // Applied is how many resources the apply reported on, and whether any of them moved. + Applied int `json:"applied,omitempty"` + Changed bool `json:"changed,omitempty"` + + // Running is the substrate's containers, confirmed up. + Running []string `json:"running,omitempty"` + // Answered is what the control plane said back — not merely that it is up. + Answered string `json:"control-plane,omitempty"` + + // Stopped names why a dry run went no further. Empty on a real run. + Stopped string `json:"stopped,omitempty"` +} + +// Run performs the bootstrap, saying what it is doing as it goes. +// +// Every step is idempotent, and every step says whether it found something or changed it. That is +// not politeness: this program is run repeatedly while somebody gets a machine working, and a step +// that cannot tell "already done" from "just done" makes the second run indistinguishable from the +// first — which is how a person stops believing any of it. +// +// It does not retry. A pull that failed for a reason that goes away by itself is real, and the +// answer to it is to run this again: re-running is the retry, and it is one a person chooses after +// reading which step failed and why. +// +// **What this does NOT do: enrolment, the module catalogue, and assignment.** It stops at a running +// substrate with a control plane that replies — a mesh of one node with nothing joined to it. See +// the marker at the end. +func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) { + if say == nil { + say = func(string) {} + } + result := Result{DryRun: o.DryRun} + + // ---- 1. preflight ------------------------------------------------------------------- + say("preflight — what has to be true before anything is changed") + template, err := Preflight(ctx, o, d, say) + if err != nil { + return result, failed(StepPreflight, err) + } + + // Which half of the host applies things here. Asked of the machine and proved, because + // `mesh-host` pins this at link time and an installer run by hand has no link time. + sys, err := WorkOutSystem(ctx, d.Run, o.System) + if err != nil { + return result, failed(StepPreflight, err) + } + result.System = sys.Name() + say(" system " + sys.Name()) + + // ---- 2. load ------------------------------------------------------------------------ + say("load — the control plane's image, carried in this installer") + loaded, err := Load(ctx, d.Run, o.DryRun, say) + if err != nil { + return result, failed(StepLoad, err) + } + result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held + + // ---- 3. bundle ---------------------------------------------------------------------- + say("bundle — what this machine will be asked to be") + rewritten, err := Rewrite(template, loaded.ID) + if err != nil { + return result, failed(StepBundle, err) + } + result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out + if rewritten.Changed { + say(fmt.Sprintf(" control plane %s", rewritten.Now)) + say(fmt.Sprintf(" replacing %s, named in %d place(s)", + rewritten.Was, rewritten.Places)) + } else { + say(fmt.Sprintf(" control plane %s — the template already named it, nothing rewritten", + rewritten.Now)) + } + for _, kept := range rewritten.Kept { + say(" left alone " + kept) + } + if rewritten.BrokerAddress != "" { + // Said every time, and never changed. Every enrolment token this mesh issues will tell a + // joining node to dial this address, and a wrong one is silent until the second node fails + // to come back. The installer does not know this machine's address and will not invent it. + say(" nodes will dial " + rewritten.BrokerAddress + + " — check this is an address other machines can reach") + } + + if o.DryRun { + // Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it + // if wrong while changing nothing. The bundle has been produced and re-parsed in memory, + // which is everything that could be checked without touching the machine; what is left is + // loading, applying and asking the result questions, and none of those can be answered by + // not doing them. + say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources)) + result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + + "loaded or applied" + say("\n" + result.Stopped) + return result, nil + } + + if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil { + return result, failed(StepBundle, err) + } + result.BundleWrote = true + say(fmt.Sprintf(" wrote %s (%d resources) — read it, this is what is applied", + o.Out, rewritten.Resources)) + + // ---- 4. apply ----------------------------------------------------------------------- + say("apply — raising the substrate") + report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, o.Out, d.Run, say) + result.Applied, result.Changed = len(report.Outcomes), report.Changed() + if err != nil { + return result, failed(StepApply, err) + } + if report.Changed() { + say(fmt.Sprintf(" applied %d resource(s)", len(report.Outcomes))) + } else { + say(fmt.Sprintf(" already matches %d resource(s) checked, nothing moved", + len(report.Outcomes))) + } + + // ---- 5. verify ---------------------------------------------------------------------- + say("verify — the substrate is up, and the control plane replies") + verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say) + result.Running, result.Answered = verified.Running, verified.Answered + if err != nil { + return result, failed(StepVerify, err) + } + + say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + + // NEXT STAGE — NOT IMPLEMENTED HERE. + // + // What remains between "a mesh exists" and "a mesh does something": issuing this machine a + // token and enrolling it as its own first node, registering the module catalogue with the + // control plane, and assigning modules to nodes. All three are conversations with the control + // plane that has just been proved to reply, so they belong after this point and inside none of + // the steps above. + // + // Left out rather than half-written. Everything above changes a machine; all of that changes a + // mesh, and a program that did both would have two jobs and one name. + say("not done here: enrolment, the module catalogue, and assignment.") + + return result, nil +} diff --git a/internal/bootstrap/load.go b/internal/bootstrap/load.go new file mode 100644 index 0000000..a44db70 --- /dev/null +++ b/internal/bootstrap/load.go @@ -0,0 +1,122 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-host/internal/image" +) + +// Loaded is the control plane's image on this machine. +type Loaded struct { + // ID is what the bundle will name the image by: sha256 of its own configuration. + ID string + // Tags is what it was called when it was saved. For a person, never for the bundle. + Tags []string + // Held is true when the machine already had it and nothing was loaded. + Held bool +} + +// Load puts the carried control-plane image into this machine's container runtime. +// +// **Idempotent by asking first, which is possible because the id is a fact about the file.** The +// image id is read out of the saved tar (see `internal/image`.ID) before the runtime is asked +// anything, so this can ask "do you already hold exactly this image" — and on the second, third +// and tenth run of the installer the answer is yes and nothing is loaded. A load that scraped the +// id out of what `docker load` printed could only know that after loading, so it would load every +// time and report the same thing either way. +// +// It reads back (novox/hq ADR 0018). A load that reported success and left nothing there is a +// failure, not a convergence, and the apply would then meet a bundle naming an image the machine +// does not hold — which fails correctly but two steps too late. +func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loaded, error) { + saved, err := image.Saved() + if err != nil { + return Loaded{}, err + } + return loadImage(ctx, run, saved, dryRun, say) +} + +// loadImage is Load with the carried bytes handed in, so the whole path can be tested against a +// saved image a test builds rather than against whatever a particular build embedded. +func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say func(string)) (Loaded, error) { + id, err := image.ID(saved) + if err != nil { + return Loaded{}, err + } + loaded := Loaded{ID: id, Tags: image.Tags(saved)} + + if held, err := holdsImage(ctx, run, id); err != nil { + return loaded, err + } else if held { + loaded.Held = true + say(" already held " + id + " — nothing loaded") + return loaded, nil + } + + if dryRun { + say(fmt.Sprintf(" would load %s (%d bytes)", id, len(saved))) + return loaded, nil + } + + // Through a file rather than through stdin: the runner this repository shares runs a command + // and captures its output, and giving it a second mouth for one caller would change every + // applier's contract for the sake of one step (internal/apply's Runner). + tarball, err := os.CreateTemp("", "mesh-control-*.tar") + if err != nil { + return loaded, fmt.Errorf("nowhere to put the carried image while loading it: %w", err) + } + defer os.Remove(tarball.Name()) + + if _, err := tarball.Write(saved); err != nil { + tarball.Close() + return loaded, fmt.Errorf("cannot write the carried image to %s: %w", tarball.Name(), err) + } + if err := tarball.Close(); err != nil { + return loaded, fmt.Errorf("cannot finish writing %s: %w", tarball.Name(), err) + } + + if _, err := run(ctx, "docker", "load", "--input", tarball.Name()); err != nil { + return loaded, fmt.Errorf( + "the container runtime would not load the carried control-plane image: %w", err) + } + + // Read back. This is what makes "loaded" a fact rather than an intention. + held, err := holdsImage(ctx, run, id) + if err != nil { + return loaded, err + } + if !held { + return loaded, fmt.Errorf( + "the load reported success and this machine does not hold %s.\n"+ + "The bundle names the control plane by that id and nothing serves it, so the "+ + "apply would refuse. Check what `docker load` actually took", id) + } + say(" loaded " + id) + return loaded, nil +} + +// holdsImage asks the runtime whether this exact image is present. +// +// It asks for the id back rather than reading the exit code, because an image inspected by id and +// an image inspected by a tag that happens to point somewhere else are the same successful +// command. What is wanted is "this one", and the answer says which one. +func holdsImage(ctx context.Context, run Runner, id string) (bool, error) { + out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", id) + if err != nil { + // Absent is an answer, not a failure. Every other reason the runtime might refuse looks + // the same from here — which is why preflight proves the runtime answers before this runs, + // rather than this trying to tell the two apart from an exit code. + return false, nil + } + got := strings.TrimSpace(out) + if got != id { + return false, fmt.Errorf( + "asked for image %s, the runtime answered %q. An image id is the digest of the "+ + "image's own configuration, so these are two different images and the bundle "+ + "would name the wrong one", id, got) + } + return true, nil +} diff --git a/internal/bootstrap/load_test.go b/internal/bootstrap/load_test.go new file mode 100644 index 0000000..0442875 --- /dev/null +++ b/internal/bootstrap/load_test.go @@ -0,0 +1,211 @@ +package bootstrap + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/image" +) + +// Docker is never required here. What is being tested is which commands the installer issues and +// what it concludes from the answers, so the runtime is injected the way `internal/apply` injects +// its Runner (novox/hq ADR 0017). Behaviour against a real runtime is proved in the lab. + +// asked records every command, so a test can assert that something was NOT run — which is the +// whole of what idempotence means here. +type asked struct { + commands []string + answer func(name string, args []string) (string, error) +} + +func (a *asked) run(_ context.Context, name string, args ...string) (string, error) { + a.commands = append(a.commands, strings.TrimSpace(name+" "+strings.Join(args, " "))) + if a.answer == nil { + return "", errors.New("this test did not expect any command to be run") + } + return a.answer(name, args) +} + +func (a *asked) ran(fragment string) bool { + for _, command := range a.commands { + if strings.Contains(command, fragment) { + return true + } + } + return false +} + +func savedImageFixture(t *testing.T, digest string) []byte { + t.Helper() + entries, err := json.Marshal([]struct { + Config string + RepoTags []string + }{{Config: digest + ".json", RepoTags: []string{"mesh-control:test"}}}) + if err != nil { + t.Fatal(err) + } + + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + if err := writer.WriteHeader(&tar.Header{ + Name: "manifest.json", Mode: 0o644, Size: int64(len(entries)), + }); err != nil { + t.Fatal(err) + } + if _, err := writer.Write(entries); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +const fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333" + +// A machine that already holds the image is not loaded again, and says so. +// +// This is the idempotence the installer's usefulness rests on: it is run over and over while +// somebody gets a machine working, and a step that did its work again every time would be +// indistinguishable from one that had never run. +func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "sha256:" + fixtureDigest + "\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + + if !loaded.Held { + t.Error("the machine already held the image and the load did not say so") + } + if runtime.ran("docker load") { + t.Errorf("the image was loaded again although the machine held it: %v", runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "already held") { + t.Errorf("nothing was said about finding the image already there: %v", said) + } +} + +// The id comes out of the file, and the bundle is named by it. +// +// Not scraped from what `docker load` prints — that is a sentence for a person, which reads +// `Loaded image: name:tag` or `Loaded image ID: sha256:…` depending on how the image was saved. +// A program depending on which one a runtime chose would be depending on a runtime version. +func TestTheImageIdComesFromTheCarriedFileNotFromWhatTheRuntimeSays(t *testing.T) { + runtime := &asked{} + inspected := 0 + runtime.answer = func(_ string, args []string) (string, error) { + switch { + case len(args) > 1 && args[0] == "image" && args[1] == "inspect": + inspected++ + if inspected == 1 { + return "", errors.New("Error: No such image") + } + return "sha256:" + fixtureDigest + "\n", nil + case len(args) > 0 && args[0] == "load": + // Deliberately says something else entirely. The id must not come from here. + return "Loaded image: some-other-name:whatever\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + } + + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("the image id is %q, want sha256:%s", loaded.ID, fixtureDigest) + } + if loaded.Held { + t.Error("an image that had to be loaded was reported as already held") + } + if !runtime.ran("docker load") { + t.Errorf("the image was never loaded: %v", runtime.commands) + } +} + +// A load that reported success and left nothing there is a failure, not a convergence +// (novox/hq ADR 0018). Without the read-back it would surface later as the host refusing a bundle +// naming an image nothing serves — a true message about the wrong thing. +func TestALoadThatLeftNothingBehindIsAFailure(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "Loaded image: mesh-control:test\n", nil + }} + + _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}) + if err == nil { + t.Fatal("a load that left nothing on the machine was reported as success") + } + if !strings.Contains(err.Error(), fixtureDigest) { + t.Errorf("the failure does not say which image is missing: %v", err) + } +} + +// A dry run changes nothing, and still knows the id — because the id is a property of the carried +// file. That is what lets `--dry-run` produce and check the real bundle rather than a guess. +func TestADryRunLearnsTheIdAndLoadsNothing(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "", fmt.Errorf("a dry run ran %v", args) + }} + + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), true, func(string) {}) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("a dry run did not work out the image id: %q", loaded.ID) + } + if runtime.ran("docker load") { + t.Errorf("a dry run loaded an image: %v", runtime.commands) + } +} + +// An installer built from a plain checkout carries no image, and says which build step is missing. +// Discovered here, before a machine is touched, rather than after a bundle has been written. +func TestAnInstallerCarryingNoImageSaysSoRatherThanRaisingHalfAMesh(t *testing.T) { + if !image.IsEmpty() { + t.Skip("this checkout has a saved image embedded") + } + _, err := Load(context.Background(), (&asked{}).run, false, func(string) {}) + if !errors.Is(err, image.ErrEmpty) { + t.Fatalf("an installer with no control-plane image gave %v, want ErrEmpty", err) + } + if !strings.Contains(err.Error(), "make bootstrap") { + t.Errorf("the refusal does not say how to build one that carries an image: %v", err) + } +} + +// Two different images cannot share an id, so an answer that is not the id asked about means the +// runtime is talking about something else. Reported rather than believed. +func TestARuntimeAnsweringAboutADifferentImageIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, _ []string) (string, error) { + return "sha256:" + strings.Repeat("9", 64) + "\n", nil + }} + if _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil { + t.Fatal("the runtime answered about a different image and it was accepted") + } +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go new file mode 100644 index 0000000..476a530 --- /dev/null +++ b/internal/bootstrap/preflight.go @@ -0,0 +1,211 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/image" + "github.com/novox/mesh-host/internal/profile" +) + +// DefaultRegistry is where an image reference that names no host comes from. +const DefaultRegistry = "registry-1.docker.io:443" + +// Preflight refuses early and plainly, and returns the bundle template it read. +// +// Everything here is a thing that will otherwise be discovered half way through: a machine with +// no runtime found after a bundle has been written, a template that does not parse found after an +// image has been loaded, a registry that cannot be reached found inside a `docker pull` that +// reports a network error and not a missing image. The order is cheapest first, so a mistake in +// what the installer was pointed at costs nothing to find. +func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte, error) { + // 1. Does this installer carry what it claims to? + // + // Asked before the machine is touched, for the same reason `mesh-host bundle` exists: a host + // that carries no substrate must say so when somebody asks, not on a first node + // (internal/bundle). An installer built without an image would otherwise get a machine as far + // as a running store and a running broker and stop. + if image.IsEmpty() { + return nil, image.ErrEmpty + } + saved, err := image.Saved() + if err != nil { + return nil, err + } + carriedID, err := image.ID(saved) + if err != nil { + return nil, err + } + say(fmt.Sprintf(" control plane %s carried (%s)", + firstOr(image.Tags(saved), "untagged"), carriedID)) + + // 2. Is the template there, and is it a substrate? + template, err := os.ReadFile(o.Template) + if err != nil { + return nil, fmt.Errorf( + "the bundle template could not be read: %w\n"+ + "It is what this machine will be asked to be, so there is nothing to do without "+ + "it. Point --bundle at one; mesh-host's examples/substrate-first-node.lock is "+ + "the shape", err) + } + // Parsed here as well as at the rewrite, because a template that is not a declaration should + // cost a second rather than an image load and a written file. + parsed, err := declaration.ParseFileTrusted(template) + if err != nil { + return nil, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + if _, err := controlPlaneIn(parsed); err != nil { + return nil, err + } + say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources))) + + // 3. Does a container runtime ANSWER? + // + // Not "is it installed" — novox/hq 04-ISSUES/007 is exactly that mistake, and the detector + // this uses is the one written for it: it asks the daemon for its server version, which fails + // when the daemon is down however complete the installation is. + // + // **Yes, the bundle installs the runtime itself**, and that is not a contradiction. The + // installer needs one BEFORE the apply, because the control plane's image is loaded into it + // first; the bundle still declares the package and the service because the host must own them + // and reassert them at every reconcile. So this is not a duplicate check — it is the one thing + // the bootstrap cannot bootstrap. + // + // Polled rather than asked once. A socket-activated daemon queued behind + // `network-online.target` is not absent, it is a few seconds away, and `docker load` against + // one blocks silently rather than failing (04-ISSUES/024). Waiting is the honest reading. + if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil { + return nil, err + } + + // 4. Can this machine reach what the bundle's images come from? + // + // Asked of the hosts the bundle actually names rather than of the internet in general. The + // mesh's own image is carried and needs nothing served — it is skipped here for exactly that + // reason. Everything else is somebody else's image at somebody else's registry, and a machine + // that cannot reach it fails inside a pull, which reports a network error where a person + // reads a missing image. + for _, host := range registriesIn(parsed) { + dialing, cancel := context.WithTimeout(ctx, o.Timeout) + err := d.Dial(dialing, host) + cancel() + if err != nil { + return nil, fmt.Errorf( + "this machine cannot reach %s, and the bundle's images are served from there: "+ + "%w\nThe apply would fail inside a pull, which says the wrong thing. Fix the "+ + "machine's network, or point the bundle at a registry it can reach", + host, err) + } + say(" reachable " + host) + } + return template, nil +} + +// waitForRuntime asks the runtime, repeatedly, until it answers or the wait runs out. +func waitForRuntime(ctx context.Context, run Runner, probe, wait time.Duration, say func(string)) error { + detector := containerRuntimeDetector(run) + + deadline := time.Now().Add(wait) + var last string + for { + probing, cancel := context.WithTimeout(ctx, probe) + verdict := detector.Detect(probing) + cancel() + if verdict.Present { + say(" container runtime " + verdict.Detail) + return nil + } + last = verdict.Detail + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(runtimeAskEvery): + } + } + return fmt.Errorf( + "this machine has no container runtime that answers, after waiting %s: %s\n"+ + "An installed package is not a capability (novox/hq 04-ISSUES/007) — the daemon was "+ + "asked and did not reply. Start it, then run this again; every step is idempotent", + wait, last) +} + +// runtimeAskEvery is how often the runtime is asked again while waiting for it. +var runtimeAskEvery = 2 * time.Second + +// containerRuntimeDetector is the host's OWN detector for a working runtime, not a second +// implementation of the same question. Two answers to "is there a container runtime here" is how +// the installer and the host come to disagree about a machine. +func containerRuntimeDetector(run Runner) profile.Detector { + for _, detector := range profile.Default(profile.Runner(run)) { + if detector.Name() == profile.CapContainerRuntime { + return detector + } + } + // Unreachable unless the host's own detector set loses its container runtime, which would be + // a change nobody would make on purpose — said rather than nil-dereferenced. + panic("the host detects no container runtime capability, and the installer needs that answer") +} + +// registriesIn is every host the bundle's images would be fetched from, without duplicates and in +// the order they appear. +func registriesIn(d *declaration.Declaration) []string { + var hosts []string + seen := map[string]bool{} + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok { + continue + } + host, served := registryOf(container.Image) + if !served || seen[host] { + continue + } + seen[host] = true + hosts = append(hosts, host) + } + return hosts +} + +// registryOf says where an image would be fetched from, and whether anything has to serve it. +// +// The second return is false for an image named by the digest of its own configuration: nothing +// serves those and nothing can (see `internal/declaration`'s checkImage). That is the whole reason +// the mesh's own control plane can be raised on a machine with no registry anywhere. +// +// The rule for the rest is the container runtime's own: the part before the first slash is a +// registry host if it looks like one — it has a dot, or a port, or it is `localhost` — and +// otherwise it is part of a repository name on the default registry. +func registryOf(reference string) (string, bool) { + if reference == "" || strings.HasPrefix(reference, "sha256:") { + return "", false + } + name := reference + if at := strings.Index(name, "@"); at >= 0 { + name = name[:at] + } + + first, _, hasPath := strings.Cut(name, "/") + if !hasPath || !(strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost") { + return DefaultRegistry, true + } + if !strings.Contains(first, ":") { + // A registry with no port is reached over HTTPS, which is where a pull would go. + return first + ":443", true + } + return first, true +} + +func firstOr(values []string, fallback string) string { + if len(values) == 0 || strings.TrimSpace(values[0]) == "" { + return fallback + } + return values[0] +} diff --git a/internal/bootstrap/preflight_test.go b/internal/bootstrap/preflight_test.go new file mode 100644 index 0000000..627dce0 --- /dev/null +++ b/internal/bootstrap/preflight_test.go @@ -0,0 +1,126 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// An installed package is not a capability (novox/hq 04-ISSUES/007). The daemon is asked, and a +// machine where it does not answer is refused before anything is loaded, written or applied. +// +// The refusal has to be plain, because the person reading it is standing in front of a machine +// that will not work: it says what was asked, what came back, that re-running is safe, and names +// the record that explains why an installed docker is not enough. +func TestPreflightRefusesPlainlyWhenTheRuntimeDoesNotAnswer(t *testing.T) { + silent := func(context.Context, string, ...string) (string, error) { + return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock") + } + + // No wait, so this is one attempt: what is being tested is the refusal, not the patience. + err := waitForRuntime(context.Background(), silent, time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a machine whose container runtime does not answer was accepted") + } + for _, wanted := range []string{ + "no container runtime that answers", + "Cannot connect to the Docker daemon", + "04-ISSUES/007", + "idempotent", + } { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a runtime that is merely slow to start is waited for rather than refused. +// +// A socket-activated daemon queued behind the network is not absent, it is a few seconds away. +// Refusing on the first attempt would make a correct bootstrap fail for being observed too early — +// and `docker load` against such a daemon blocks silently rather than failing, which is how one +// became a 35-minute silence (04-ISSUES/024). +func TestARuntimeThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := runtimeAskEvery + runtimeAskEvery = time.Millisecond + defer func() { runtimeAskEvery = previous }() + + attempts := 0 + slow := func(context.Context, string, ...string) (string, error) { + attempts++ + if attempts < 3 { + return "", errors.New("Cannot connect to the Docker daemon") + } + return "27.0.3\n", nil + } + + var said []string + if err := waitForRuntime(context.Background(), slow, time.Second, time.Second, + func(line string) { said = append(said, line) }); err != nil { + t.Fatalf("a runtime that answered on the third ask was refused: %v", err) + } + if attempts != 3 { + t.Errorf("the runtime was asked %d time(s)", attempts) + } + if !strings.Contains(strings.Join(said, "\n"), "27.0.3") { + t.Errorf("the version the daemon reported was not said back: %v", said) + } +} + +// What has to be reachable is what the bundle actually names, not "the internet". +// +// The mesh's own image is carried and nothing serves it, so asking a registry about it would be +// asking a question with no answer — which is the whole point of naming an image by the digest of +// its own configuration. +func TestOnlyTheRegistriesTheBundleNamesAreAskedAbout(t *testing.T) { + parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"}, + {"id":"broker","type":"container","name":"mesh-broker","image":"192.0.2.250:5000/lavinmq@sha256:` + + strings.Repeat("8", 64) + `"}, + {"id":"control-plane","type":"container","name":"mesh-control","image":"` + held + `"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + got := registriesIn(parsed) + want := []string{DefaultRegistry, "192.0.2.250:5000"} + if len(got) != len(want) { + t.Fatalf("asked about %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("asked about %v, want %v", got, want) + } + } +} + +func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) { + // The container runtime's own rule: the part before the first slash is a registry host if it + // has a dot, a port, or is localhost. Getting this wrong means dialling a hostname that is + // really the first half of a repository name, and refusing a machine that is fine. + for _, c := range []struct { + reference string + host string + served bool + }{ + {"postgres@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"cloudamqp/lavinmq@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"192.0.2.250:5000/postgres@sha256:" + strings.Repeat("a", 64), "192.0.2.250:5000", true}, + {"localhost/mesh-control@sha256:" + strings.Repeat("a", 64), "localhost:443", true}, + {"registry.example.com/a/b@sha256:" + strings.Repeat("a", 64), "registry.example.com:443", true}, + // Held by this machine. Nothing serves it, and nothing can. + {"sha256:" + strings.Repeat("a", 64), "", false}, + {"", "", false}, + } { + host, served := registryOf(c.reference) + if host != c.host || served != c.served { + t.Errorf("%q → (%q, %v), want (%q, %v)", c.reference, host, served, c.host, c.served) + } + } +} diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go new file mode 100644 index 0000000..2cdcde5 --- /dev/null +++ b/internal/bootstrap/rewrite.go @@ -0,0 +1,241 @@ +package bootstrap + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// ControlPlaneID is the resource the installer replaces the image of. +// +// A resource id rather than a container name or a guess at the image, because the id is the one +// thing a declaration promises is stable — it is what lets the store say *this is the same +// resource I applied last time* (`internal/declaration`, Resource.Identity). A bundle that does not +// name one is refused rather than applied without a control plane, which would raise a store and a +// broker and no mesh. +const ControlPlaneID = "control-plane" + +// brokerAddressVar is what a token tells an enrolling node to dial. +// +// Not rewritten here — see the note in Rewrite — but reported, because it is the field most likely +// to be wrong on a machine that is not the one the template was written for, and it is wrong in a +// way nothing notices until a second node tries to join. +const brokerAddressVar = "MESH_BROKER_ADDRESS" + +// Rewritten is the bundle this machine will apply, and what was done to produce it. +type Rewritten struct { + // Bundle is the produced file's bytes — the template with one image reference replaced, + // comments and all. + Bundle []byte + // Declaration is that bundle, parsed. Carried so the apply and the verify are talking about + // the same document rather than each re-reading the file and hoping. + Declaration *declaration.Declaration + + // Was is the image the template named the control plane by; Now is the one it names it by. + Was string + Now string + // Places is how many times that reference appeared, and therefore how many were replaced. + Places int + // Changed is false when the template already named this image — a re-run on a bundle this + // installer produced earlier. + Changed bool + + // Kept is every other container image, unchanged, as " ". Reported rather than + // assumed: "postgres was left alone" is a claim, and this is the evidence for it. + Kept []string + + // Resources is how many things the produced bundle asks for. + Resources int + // BrokerAddress is what the control plane will tell enrolling nodes to dial, or empty. + BrokerAddress string +} + +// Rewrite produces the bundle this machine will apply from the template it was given. +// +// **One substitution, and it is textual.** The control plane's image becomes the id of the image +// this machine now holds; nothing else changes. Textual rather than parse-and-re-serialise because +// the produced file has to be *read* — a person getting a machine working must be able to open it, +// see the substrate they recognise, and see exactly one thing different. Re-serialising a parsed +// declaration would drop every comment in the template, and those comments are where the reasons +// live. +// +// **Every place that reference appears, not only the container.** The bundle names the control +// plane's image twice: once as the container that runs `serve`, and once inside the action that +// runs `migrate` to create the contexts' schemas. Replacing only the container would leave the +// migration pointing at an image no registry serves, and the apply would fail in the middle — +// after the store is up, before the broker. They are one image and they move together. +// +// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references +// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is +// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other +// container's image is compared against what it was. +// +// **What this deliberately does NOT rewrite:** MESH_BROKER_ADDRESS, the endpoint every enrolment +// token will carry. It differs per machine and it is silently fatal when wrong — a node enrols +// against a dead address and nothing complains until it fails to come back. It belongs to the +// enrolment stage, which is not built yet, so this reports it loudly and leaves it alone rather +// than guessing an address for a machine it has not been told about. +func Rewrite(template []byte, imageID string) (Rewritten, error) { + if !isImageID(imageID) { + return Rewritten{}, fmt.Errorf( + "%q is not an image id. The control plane is named by the digest of its own "+ + "configuration — sha256: and sixty-four hex characters — because nothing serves "+ + "it and there is no manifest digest to use instead", imageID) + } + + before, err := declaration.ParseFileTrusted(template) + if err != nil { + return Rewritten{}, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + control, err := controlPlaneIn(before) + if err != nil { + return Rewritten{}, err + } + + out := Rewritten{Was: control.Image, Now: imageID, BrokerAddress: control.Env[brokerAddressVar]} + + occurrences := bytes.Count(template, []byte(control.Image)) + if occurrences == 0 { + // The parser found the image and the bytes do not contain it, which means the two are + // reading different things. Refused rather than replaced-zero-times-and-reported-success. + return Rewritten{}, fmt.Errorf( + "the control plane's image is %q according to the parsed template, and that text is "+ + "not in the file. Nothing was rewritten", control.Image) + } + out.Places = occurrences + + switch { + case control.Image == imageID: + // Already this image. The idempotent case, and the one that happens whenever somebody + // re-runs the installer against a bundle it produced earlier. + out.Bundle = template + default: + out.Bundle = bytes.ReplaceAll(template, []byte(control.Image), []byte(imageID)) + out.Changed = true + } + + // Read back, on the bytes that will actually be applied. Everything above is an intention + // until the produced file is parsed and asked what it says. + after, err := declaration.ParseFileTrusted(out.Bundle) + if err != nil { + return Rewritten{}, fmt.Errorf( + "the bundle this produced is not a declaration, so the substitution broke it: %w", err) + } + out.Declaration, out.Resources = after, len(after.Resources) + + produced, err := controlPlaneIn(after) + if err != nil { + return Rewritten{}, err + } + if produced.Image != imageID { + return Rewritten{}, fmt.Errorf( + "the produced bundle still names the control plane %q, not %q", produced.Image, imageID) + } + + // And nothing else moved. A substitution on text can in principle catch more than it was + // aimed at, and "postgres was left exactly as it was" is the claim this checks rather than + // asserts. + was := containerImages(before) + for id, image := range containerImages(after) { + if id == ControlPlaneID { + continue + } + if was[id] != image { + return Rewritten{}, fmt.Errorf( + "rewriting the control plane's image also changed %q, from %q to %q. Only the "+ + "mesh's own image may move; everything else is somebody else's image at "+ + "somebody else's registry", id, was[id], image) + } + out.Kept = append(out.Kept, fmt.Sprintf("%s %s", id, image)) + } + sortStrings(out.Kept) + return out, nil +} + +// controlPlaneIn finds the container this installer replaces the image of. +func controlPlaneIn(d *declaration.Declaration) (*declaration.Container, error) { + for _, r := range d.Resources { + if r.Identity() != ControlPlaneID { + continue + } + container, ok := r.(*declaration.Container) + if !ok { + return nil, fmt.Errorf( + "this bundle's %q is a %s, and the control plane has to be a container for its "+ + "image to be named. Nothing was rewritten", ControlPlaneID, r.Kind()) + } + return container, nil + } + return nil, fmt.Errorf( + "this bundle names no %q, so there is no control plane to give this machine's image to. "+ + "A substrate without one raises a store and a broker and no mesh. It declares: %s", + ControlPlaneID, strings.Join(identities(d), ", ")) +} + +func containerImages(d *declaration.Declaration) map[string]string { + images := map[string]string{} + for _, r := range d.Resources { + if container, ok := r.(*declaration.Container); ok { + images[container.ID] = container.Image + } + } + return images +} + +func identities(d *declaration.Declaration) []string { + var ids []string + for _, r := range d.Resources { + ids = append(ids, r.Identity()) + } + return ids +} + +// isImageID is the same shape `internal/declaration` accepts for an image the machine holds. Asked +// here as well so the refusal names the installer's own mistake, rather than surfacing as a +// declaration refusal about a bundle this program wrote. +func isImageID(s string) bool { + const prefix = "sha256:" + if !strings.HasPrefix(s, prefix) || len(s) != len(prefix)+64 { + return false + } + for _, c := range s[len(prefix):] { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func sortStrings(values []string) { + for i := 1; i < len(values); i++ { + for j := i; j > 0 && values[j] < values[j-1]; j-- { + values[j], values[j-1] = values[j-1], values[j] + } + } +} + +// writeBundleFile puts the produced bundle where a person can read it, creating the directory it +// lives in. +// +// 0644, and that is deliberate: this file names an image and describes a substrate, and it holds +// the bootstrap credentials the template happens to carry — which are the same ones anybody can +// read in the template itself. It is meant to be read. What must not be world-readable is the +// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`). +func writeBundleFile(path string, content []byte) error { + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err) + } + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf( + "cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+ + "applying something nobody can read afterwards is how a machine becomes a mystery", + path, err) + } + return nil +} diff --git a/internal/bootstrap/rewrite_test.go b/internal/bootstrap/rewrite_test.go new file mode 100644 index 0000000..11798ae --- /dev/null +++ b/internal/bootstrap/rewrite_test.go @@ -0,0 +1,223 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +const ( + held = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + otherHeld = "sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +// theRealBundle is this repository's own substrate example, used rather than a fixture. +// +// A fixture would agree with whatever this code does. The example is what an installer is actually +// pointed at, it names the control plane twice, and it is the file that changes when the substrate +// changes — so a rewrite that stops working on it is a rewrite that has stopped working. +func theRealBundle(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile("../../examples/substrate-first-node.lock") + if err != nil { + t.Fatalf("reading the substrate example: %v", err) + } + return raw +} + +func TestTheControlPlaneIsNamedByTheImageThisMachineHolds(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if !out.Changed { + t.Error("the rewrite reported nothing changed, and the template named a registry image") + } + control, err := controlPlaneIn(out.Declaration) + if err != nil { + t.Fatal(err) + } + if control.Image != held { + t.Errorf("the control plane is %q, want %q", control.Image, held) + } +} + +// **Every place the bundle names that image, not only the container.** +// +// The substrate names the control plane's image twice: the container that runs `serve`, and the +// action that runs `migrate` to create the contexts' schemas. Rewriting only the container leaves +// the migration pointing at an image no registry serves, and the apply dies in the middle — after +// the store is up and before the broker. This is the test that would have caught that. +func TestEveryPlaceTheBundleNamesTheControlPlaneIsRewritten(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.Places < 2 { + t.Fatalf("the control plane's image was found in %d place(s); the substrate names it in "+ + "the container AND in the migration action", out.Places) + } + if remaining := strings.Count(string(out.Bundle), out.Was); remaining != 0 { + t.Errorf("the produced bundle still names %q in %d place(s)", out.Was, remaining) + } + if got := strings.Count(string(out.Bundle), held); got != out.Places { + t.Errorf("the produced bundle names the held image %d time(s), and %d were replaced", + got, out.Places) + } +} + +// Third-party images are somebody else's, at somebody else's registry, and the installer has no +// business touching them (novox/hq ADR 0006). +func TestPostgresAndTheBrokerAreLeftExactlyAsTheyWere(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + + produced := containerImages(out.Declaration) + for _, id := range []string{"store", "broker"} { + image, named := produced[id] + if !named { + t.Fatalf("the substrate example no longer declares a %q container", id) + } + // Compared against the template's own text rather than against an expectation written + // here: what is being defended is "unchanged", and the template is the only thing that + // knows what it said. + if !strings.Contains(string(template), `"image": "`+image+`"`) { + t.Errorf("%s is now %q, which the template does not say", id, image) + } + if strings.HasPrefix(image, "sha256:") { + t.Errorf("%s was rewritten to an image this machine holds, and nothing holds it", id) + } + } + + // And the claim in the report is the same claim, so a person reading it is reading evidence. + if len(out.Kept) != 2 { + t.Errorf("the rewrite reports %d untouched image(s): %v", len(out.Kept), out.Kept) + } +} + +// A bundle with no control plane raises a store and a broker and no mesh. Refused, because +// applying it would succeed and leave a machine that looks bootstrapped. +func TestABundleThatNamesNoControlPlaneIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"} + ]}`) + + _, err := Rewrite(template, held) + if err == nil { + t.Fatal("a bundle with no control plane was rewritten and would have been applied") + } + // The refusal has to be actionable: it says what the bundle DID declare, so somebody can see + // they pointed it at the wrong file or misspelled the id. + if !strings.Contains(err.Error(), ControlPlaneID) || !strings.Contains(err.Error(), "store") { + t.Errorf("the refusal names neither what was wanted nor what was there: %v", err) + } +} + +func TestAControlPlaneThatIsNotAContainerIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"control-plane","type":"package","package":"mesh-control"} + ]}`) + if _, err := Rewrite(template, held); err == nil { + t.Fatal("a control plane declared as a package was accepted, and a package has no image") + } +} + +// Idempotence. This program is run over and over while somebody gets a machine working, and the +// second run must be able to say the bundle already names this image rather than reporting a +// rewrite it did not perform. +func TestRewritingABundleThatAlreadyNamesTheImageChangesNothing(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + second, err := Rewrite(first.Bundle, held) + if err != nil { + t.Fatal(err) + } + if second.Changed { + t.Error("re-running the rewrite reported a change, and the image was already the one held") + } + if string(second.Bundle) != string(first.Bundle) { + t.Error("re-running the rewrite produced different bytes") + } + if second.Was != held { + t.Errorf("the second run reports it replaced %q; it replaced nothing", second.Was) + } +} + +// A DIFFERENT image, though, must move — the ordinary case of a new control plane being installed +// over an old one. "Already correct" must not be the same code path as "already ran". +func TestANewImageReplacesAnOlderHeldOne(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + second, err := Rewrite(first.Bundle, otherHeld) + if err != nil { + t.Fatal(err) + } + if !second.Changed || second.Was != held || second.Now != otherHeld { + t.Errorf("a new image did not replace the old one: changed=%v was=%q now=%q", + second.Changed, second.Was, second.Now) + } +} + +// The produced bundle is meant to be READ. Re-serialising a parsed declaration would drop every +// comment in the template, and the substrate example is mostly comments — each one recording why a +// resource is the way it is, several of them paid for in the lab. +func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + const remembered = "NAMED VOLUME" + if !strings.Contains(string(out.Bundle), remembered) { + t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered) + } +} + +// The installer must refuse its own bad input in its own words, rather than writing a bundle and +// letting the host refuse a declaration somebody did not write. +func TestSomethingThatIsNotAnImageIdIsRefused(t *testing.T) { + for _, bad := range []string{ + "", + "mesh-control:latest", + "sha256:abc", + "sha256:" + strings.Repeat("1", 63), + "sha256:" + strings.Repeat("g", 64), + "mesh-control@sha256:" + strings.Repeat("1", 64), + } { + if _, err := Rewrite(theRealBundle(t), bad); err == nil { + t.Errorf("image id %q was accepted", bad) + } + } +} + +// The address every enrolment token will carry is reported and never invented. It differs per +// machine and it is silently fatal when wrong: a node enrols against a dead address and nothing +// says so until it fails to come back. +func TestTheAddressNodesWillDialIsReportedAndNotRewritten(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.BrokerAddress == "" { + t.Fatal("the substrate example no longer says what address enrolling nodes will dial") + } + if !strings.Contains(string(out.Bundle), out.BrokerAddress) { + t.Errorf("the produced bundle no longer carries %q — it was rewritten, and nothing here "+ + "knows this machine's address", out.BrokerAddress) + } +} diff --git a/internal/bootstrap/verify.go b/internal/bootstrap/verify.go new file mode 100644 index 0000000..4bed6fe --- /dev/null +++ b/internal/bootstrap/verify.go @@ -0,0 +1,167 @@ +package bootstrap + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// controlPlaneBinary is where the control plane's program lives in its own image. +// +// A path rather than a shell command, because the image is `FROM scratch` and holds one static +// binary and nothing else — no shell to invoke, nothing to interpret a command line +// (mesh-control's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer +// carries, which is why the path can be written down here. +const controlPlaneBinary = "/mesh-control" + +// answerEvery is how often the control plane is asked again while it is starting. +var answerEvery = 2 * time.Second + +// Verified is what the substrate was found to be. +type Verified struct { + // Running is every long-running container the bundle declares, confirmed up. + Running []string + // Answered is the control plane's own first words back, so the report shows the reply rather + // than asserting there was one. + Answered string +} + +// Verify proves the substrate is up and the control plane replies. +// +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once: a runtime reports a container running from the moment the +// process starts, which is before it has opened a database, before it has read its configuration, +// and before it has failed to. So the containers are checked, and then the program inside one of +// them is asked a question and has to answer it. +// +// The question is `status`, and it is chosen rather than convenient: answering it means the +// control plane opened all three of its stores from the environment the bundle gave it. A reply +// therefore proves the image runs, that `network: host` really does reach the store on this +// machine, and that the contexts' schemas migrated — the three things the steps before this were +// for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server. +// +// It waits. A control plane that is not answering yet and a control plane that will never answer +// look identical for the first few seconds, and refusing on the first attempt would make a correct +// bootstrap fail for being observed too early. +func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration, + say func(string)) (Verified, error) { + + var out Verified + + control, err := controlPlaneIn(d) + if err != nil { + return out, err + } + + for _, container := range longRunning(d) { + state, err := containerRunning(ctx, run, probe, container) + if err != nil { + return out, err + } + if !state.running { + return out, fmt.Errorf( + "the container %q is %s, not running.\n"+ + "The apply reported success, so it was created — what it did afterwards is "+ + "in `docker logs %s`", container, state.status, container) + } + out.Running = append(out.Running, container) + say(" running " + container) + } + + answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say) + if err != nil { + return out, err + } + out.Answered = answer + return out, nil +} + +func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration, + container string, say func(string)) (string, error) { + + deadline := time.Now().Add(wait) + var last error + for { + asking, cancel := context.WithTimeout(ctx, probe) + out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status") + cancel() + + answer := strings.TrimSpace(firstLineOf(out)) + switch { + case err != nil: + last = err + case answer == "": + // Exit zero and nothing said. Treated as no answer rather than as success: a program + // that returns silence is not one that has been asked anything. + last = fmt.Errorf("it exited without saying anything") + default: + say(" replies " + container + ": " + answer) + return answer, nil + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } + return "", fmt.Errorf( + "the container %q is running and the control plane in it does not answer, after waiting "+ + "%s: %v\n"+ + "Running is not replying. `status` opens this mesh's three stores, so what failed is "+ + "most likely the store or the schemas rather than the control plane itself — "+ + "`docker logs %s` says which", container, wait, last, container) +} + +type containerState struct { + running bool + status string +} + +func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) { + asking, cancel := context.WithTimeout(ctx, probe) + defer cancel() + + // Both facts in one answer, so a container that is not running is reported with what it IS + // rather than with the absence of what it should be. + out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name) + if err != nil { + return containerState{}, fmt.Errorf( + "the container %q is not there at all, and the apply reported it applied: %w", name, err) + } + running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ") + if status == "" { + status = "in a state the runtime did not name" + } + return containerState{running: running == "true", status: status}, nil +} + +// longRunning is every container the bundle expects to still be there afterwards. +// +// A run-once step has exited by design and a scheduled step has deliberately never been started +// (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the +// substrate to be something other than what it declared. +func longRunning(d *declaration.Declaration) []string { + var names []string + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok || container.RunOnce || container.Schedule != "" { + continue + } + names = append(names, container.Name) + } + return names +} + +func firstLineOf(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go new file mode 100644 index 0000000..45eb857 --- /dev/null +++ b/internal/bootstrap/verify_test.go @@ -0,0 +1,168 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +func substrate(t *testing.T) *declaration.Declaration { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Declaration +} + +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once. A runtime reports a container running from the moment its +// process starts — before it has opened a database, and before it has failed to. +func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + switch args[0] { + case "inspect": + return "true running\n", nil + case "exec": + // Up, and saying nothing. The program inside is not answering. + return "", errors.New("exit status 1") + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("every container was running, nothing answered, and the substrate was reported up") + } + for _, wanted := range []string{"mesh-control", "Running is not replying", "docker logs"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the failure does not mention %q:\n%v", wanted, err) + } + } +} + +// Exit zero and silence is not an answer either. A program that returns nothing has not been asked +// anything, and treating it as success is the same fault one level down. +func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return " \n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err == nil { + t.Fatal("a control plane that exited zero without saying anything was accepted") + } +} + +// The substrate answering is the whole point, and what it said is reported rather than asserted. +func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node, 0 waiting\n", nil + }} + + verified, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err != nil { + t.Fatal(err) + } + // Three long-running containers: the store, the broker and the control plane. The run-once and + // scheduled shapes are excluded on purpose — a step that has exited is not a fault. + want := []string{"mesh-store", "mesh-broker", "mesh-control"} + if len(verified.Running) != len(want) { + t.Fatalf("confirmed %v running, want %v", verified.Running, want) + } + for i := range want { + if verified.Running[i] != want[i] { + t.Errorf("confirmed %v running, want %v", verified.Running, want) + } + } + if verified.Answered != "1 node, 0 waiting" { + t.Errorf("the control plane's reply is reported as %q", verified.Answered) + } +} + +// A control plane that is still opening its stores is waited for, not refused. Refusing on the +// first attempt would make a correct bootstrap fail for being observed too early. +func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + attempts := 0 + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + attempts++ + if attempts < 3 { + return "", errors.New("exit status 1") + } + return "1 node\n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, time.Second, func(string) {}); err != nil { + t.Fatalf("a control plane that answered on the third ask was refused: %v", err) + } +} + +// A container that exited is named with what it IS, so somebody can go and read its logs rather +// than being told only that something is not what it should be. +func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" { + return "false exited\n", nil + } + if args[0] == "inspect" { + return "true running\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a container that had exited was reported as part of a running substrate") + } + if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { + t.Errorf("the failure does not say which container is in what state: %v", err) + } +} + +// The control plane is asked by running the binary in its own image directly, because the image is +// `FROM scratch` and has no shell for a command line to be interpreted by. +func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node\n", nil + }} + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err != nil { + t.Fatal(err) + } + if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the control plane was never asked anything: %v", runtime.commands) + } +} diff --git a/internal/image/control-plane.tar b/internal/image/control-plane.tar new file mode 100644 index 0000000..e26e4f9 --- /dev/null +++ b/internal/image/control-plane.tar @@ -0,0 +1,8 @@ +This is not a saved image. It is the placeholder that keeps this repository buildable. + +A release build replaces this file with the output of `docker save` and puts it back afterwards: + + make bootstrap IMAGE=mesh-control: + +An installer built with this file present carries no control plane, and says so in preflight +rather than getting a machine part-way to being a mesh and stopping. diff --git a/internal/image/image.go b/internal/image/image.go new file mode 100644 index 0000000..5e94196 --- /dev/null +++ b/internal/image/image.go @@ -0,0 +1,173 @@ +// Package image is the control plane's image, carried inside the installer. +// +// **Carried rather than built, and that is not an optimisation.** The mesh's forge runs on the +// mesh. A bootstrap that needed the control plane's source in order to build it would need a +// forge to fetch that source from, and the forge is one of the things the mesh raises — so the +// mesh would be required in order to raise the mesh. Embedding the image breaks that cycle the +// same way `internal/bundle` breaks it for the declaration: the installer arrives holding +// everything a bare machine has to be given, and a bare machine is given one file +// (novox/hq ADR 0005). +// +// It is also the rule the rest of tier 0 already follows. Nobody compiles `mesh-host` on the +// machine it will run on; the binary is put there. The image it raises arrives the same way. +// +// What is embedded is the output of `docker save` — a tar holding the image's config, its layers +// and a `manifest.json` naming them. The control plane's image is `FROM scratch` with one static +// binary in it (novox/hq ADR 0006), so this is tens of megabytes rather than hundreds. +package image + +import ( + "archive/tar" + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" +) + +// The saved image, replaced at release time by `make bootstrap`. +// +// What is committed here is a placeholder, for the same reason `internal/bundle` commits locks +// that are only comments: `go:embed` refuses to compile against a file that is not there, so a +// checkout with nothing embedded would not build at all — and someone reading this repository or +// running `go test ./...` would meet a compile error instead of a program. The placeholder keeps +// the tree buildable and makes the absence a thing the installer *says*, at the earliest moment +// it can, rather than a thing a compiler says to the wrong person. +// +// It is small and it is committed. A saved image is not, and `make bootstrap` puts one here for +// the length of one build and then puts the placeholder back — exactly what `make host` does with +// the bundle it embeds. +// +//go:embed control-plane.tar +var saved []byte + +// ErrEmpty means this installer carries no control-plane image. +// +// A separate error rather than a message, so the caller can refuse in preflight — before a +// machine has been touched — instead of discovering it at the load, after the runtime has been +// probed and a bundle has been written. +var ErrEmpty = errors.New( + "this mesh-bootstrap carries no control-plane image, so it cannot raise a mesh. A release " + + "build embeds one: `make bootstrap IMAGE=` in the mesh-host repository, where " + + " is a control-plane image already built from the mesh-control source") + +// IsEmpty reports whether anything was built in. +// +// It asks whether the bytes are a tar rather than comparing them against the placeholder's text, +// because the question that matters is "can this be loaded", and a truncated or corrupted embed +// answers no to that while matching no placeholder. A tar's first header carries the string +// `ustar` at offset 257 and nothing else does by accident. +func IsEmpty() bool { + const magicAt, magic = 257, "ustar" + return len(saved) < magicAt+len(magic) || string(saved[magicAt:magicAt+len(magic)]) != magic +} + +// Saved returns the embedded tar, for loading into a container runtime. +func Saved() ([]byte, error) { + if IsEmpty() { + return nil, ErrEmpty + } + return saved, nil +} + +// manifestEntry is the part of a saved image's `manifest.json` this needs. +// +// One field. The layers are the runtime's business and the repository tags are decoration — what +// is wanted is the config, because the digest of the config IS the image id. +type manifestEntry struct { + Config string `json:"Config"` + RepoTags []string `json:"RepoTags"` +} + +// ID is the image id the runtime will give this image once it is loaded, read out of the tar. +// +// **Read here rather than parsed out of what `docker load` prints.** The load prints a sentence +// for a person — `Loaded image: name:tag` or `Loaded image ID: sha256:…`, depending on whether the +// image was saved with a tag — and a program that scraped it would be depending on which of those +// a particular runtime version chose. The id is a fact about the file, available before the +// runtime is asked anything, which is also what makes the load idempotent: the installer can ask +// whether the machine already holds THIS image before loading it. +// +// An image id is the sha256 of the image's configuration document (novox/hq ADR 0006, and see +// `internal/declaration`'s checkImage). `manifest.json` names that document by its digest — as +// `<64hex>.json` in the older layout and `blobs/sha256/<64hex>` in the OCI one — so both forms +// reduce to the same sixty-four characters. +func ID(saved []byte) (string, error) { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return "", err + } + + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil { + return "", fmt.Errorf( + "the carried image has a manifest.json this cannot read, so the image it holds "+ + "cannot be named: %w", err) + } + // One image, deliberately. A tar holding several would leave the installer choosing which + // one is the control plane, and a bootstrap must not be the thing that guesses. + if len(entries) != 1 { + return "", fmt.Errorf( + "the carried image holds %d images, and the installer raises exactly one control "+ + "plane. Save a single image: `docker save --output … `", len(entries)) + } + + digest := strings.TrimSuffix(path.Base(entries[0].Config), ".json") + if !isSHA256(digest) { + return "", fmt.Errorf( + "the carried image names its configuration %q, which is not a sha256 digest. An "+ + "image id is the digest of that configuration, so there is nothing to call this "+ + "image", entries[0].Config) + } + return "sha256:" + digest, nil +} + +// Tags is what the saved image was called when it was saved, for a person reading a report. +// +// Decoration, and said so: the installer names the image by its id everywhere it matters, because +// a tag is exactly what a pinned bundle may not rely on (novox/hq ADR 0006). This is here so a +// report can say which build somebody embedded, which is otherwise sixty-four characters of hex. +func Tags(saved []byte) []string { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return nil + } + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil || len(entries) == 0 { + return nil + } + return entries[0].RepoTags +} + +func fileFromTar(archive []byte, want string) ([]byte, error) { + reader := tar.NewReader(bytes.NewReader(archive)) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf( + "the carried image has no %s, so it is not something `docker save` produced. "+ + "Embed the output of `docker save`, not a layer or a build context", want) + } + if err != nil { + return nil, fmt.Errorf("the carried image cannot be read as a tar: %w", err) + } + if path.Clean(header.Name) == want { + return io.ReadAll(reader) + } + } +} + +func isSHA256(s string) bool { + if len(s) != 64 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/image/image_test.go b/internal/image/image_test.go new file mode 100644 index 0000000..74970db --- /dev/null +++ b/internal/image/image_test.go @@ -0,0 +1,152 @@ +package image + +import ( + "archive/tar" + "bytes" + "encoding/json" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +// savedImage builds what `docker save` produces, as far as this package reads it. +func savedImage(t *testing.T, files map[string]string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + for name, content := range files { + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))} + if err := writer.WriteHeader(header); err != nil { + t.Fatalf("building the fixture: %v", err) + } + if _, err := writer.Write([]byte(content)); err != nil { + t.Fatalf("building the fixture: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("building the fixture: %v", err) + } + return buffer.Bytes() +} + +func manifest(t *testing.T, config string, tags ...string) string { + t.Helper() + raw, err := json.Marshal([]manifestEntry{{Config: config, RepoTags: tags}}) + if err != nil { + t.Fatalf("building the fixture: %v", err) + } + return string(raw) +} + +// The image id is read from the FILE, before any runtime is asked anything. +// +// That is what makes the load idempotent: knowing the id in advance lets the installer ask "do you +// already hold exactly this" instead of loading and then finding out. Scraping it from what +// `docker load` prints would only be possible after loading, so the second run of an installer +// would load again every time and be unable to say it had not. +func TestTheImageIdIsReadFromTheSavedFile(t *testing.T) { + digest := strings.Repeat("a", 64) + // Both layouts `docker save` has used. The older one names the config `.json`; the OCI + // one names it `blobs/sha256/`. They carry the same sixty-four characters, and a + // reader that understood only one would work until somebody upgraded their runtime. + for _, config := range []string{digest + ".json", "blobs/sha256/" + digest} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + id, err := ID(saved) + if err != nil { + t.Fatalf("config %q: %v", config, err) + } + if id != "sha256:"+digest { + t.Errorf("config %q gave id %q, want sha256:%s", config, id, digest) + } + } +} + +func TestTheSavedTagsAreReadForAPersonToRecognise(t *testing.T) { + saved := savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("b", 64)+".json", "mesh-control:v1"), + }) + got := Tags(saved) + if len(got) != 1 || got[0] != "mesh-control:v1" { + t.Errorf("tags = %v, want [mesh-control:v1]", got) + } +} + +// A tar that is not a saved image is refused with what is wrong, not with a nil id. +// +// The installer names the control plane by this id in the bundle it writes. An id it could not +// read, treated as empty, would produce a bundle naming nothing — refused by the host two steps +// later, with a message about a declaration rather than about what somebody embedded. +func TestSomethingThatIsNotASavedImageIsRefused(t *testing.T) { + notAnImage := savedImage(t, map[string]string{"hello": "world"}) + if _, err := ID(notAnImage); err == nil { + t.Error("a tar with no manifest.json was accepted as a saved image") + } else if !strings.Contains(err.Error(), "docker save") { + t.Errorf("the refusal does not say what to embed instead: %v", err) + } + + if _, err := ID([]byte("this is not a tar at all")); err == nil { + t.Error("bytes that are not a tar were accepted") + } +} + +// Exactly one image. A bootstrap that chose between several would be the thing that guesses which +// one is the control plane, and it would guess right until the day somebody saved two. +func TestATarHoldingSeveralImagesIsRefused(t *testing.T) { + entries, err := json.Marshal([]manifestEntry{ + {Config: strings.Repeat("a", 64) + ".json"}, + {Config: strings.Repeat("b", 64) + ".json"}, + }) + if err != nil { + t.Fatal(err) + } + saved := savedImage(t, map[string]string{"manifest.json": string(entries)}) + if _, err := ID(saved); err == nil { + t.Error("a tar holding two images was accepted") + } +} + +// An id is a digest or it is nothing. A truncated one names several images, and which one ran +// would be whichever the runtime matched first — the same reasoning `internal/declaration` gives +// for refusing a short image reference. +func TestAConfigThatIsNotADigestIsRefused(t *testing.T) { + for _, config := range []string{"config.json", "abc.json", "blobs/sha256/" + strings.Repeat("a", 63)} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + if _, err := ID(saved); err == nil { + t.Errorf("config %q was accepted and is not a digest", config) + } + } +} + +// The committed placeholder must read as "carries nothing", so an installer built from a plain +// checkout says so in preflight rather than getting a machine as far as a running store and +// stopping. This is the same guarantee `internal/bundle` makes about a lock file of only comments. +func TestAnInstallerBuiltFromAPlainCheckoutCarriesNothing(t *testing.T) { + if !IsEmpty() { + // Not a failure of this checkout: `make bootstrap` embeds a real image and puts the + // placeholder back, so a real image here means a build was interrupted. + t.Skip("this checkout has a saved image embedded, so there is no placeholder to check") + } + if _, err := Saved(); err == nil { + t.Fatal("an installer carrying only the placeholder reported it carries an image") + } +} + +// And "empty" is decided by whether the bytes could be loaded, not by matching the placeholder's +// text. A truncated or corrupted embed is equally unloadable and equally worth refusing early. +func TestEmptyMeansUnloadableRatherThanEqualToThePlaceholder(t *testing.T) { + restore := saved + defer func() { saved = restore }() + + saved = []byte("half a tar, cut off") + if !IsEmpty() { + t.Error("bytes that are not a tar were reported as a carried image") + } + + saved = savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("c", 64)+".json"), + }) + if IsEmpty() { + t.Error("a real saved image was reported as no image at all") + } +} -- 2.54.0 From 9c9e02ba1d8c54e1140ba0a2dacee62048121f7a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 10 Sep 2026 23:18:07 +0200 Subject: [PATCH 3/9] mesh-bootstrap: drop an unread parameter A parameter nothing reads is a claim the function makes about what it needs, and this one was wrong. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/bootstrap/apply.go | 2 +- internal/bootstrap/bootstrap.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go index e792059..6f8e7c0 100644 --- a/internal/bootstrap/apply.go +++ b/internal/bootstrap/apply.go @@ -33,7 +33,7 @@ type Runner = apply.Runner // clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is // not one. func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration, - source string, run Runner, say func(string)) (apply.Report, error) { + run Runner, say func(string)) (apply.Report, error) { // Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host` // does: finding out half way through is the half-configured machine tier 0 exists to prevent. diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 428aefc..44163b1 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -222,7 +222,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- 4. apply ----------------------------------------------------------------------- say("apply — raising the substrate") - report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, o.Out, d.Run, say) + report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say) result.Applied, result.Changed = len(report.Outcomes), report.Changed() if err != nil { return result, failed(StepApply, err) -- 2.54.0 From af953dbb0d2788eb595eb8346b88baba72324167 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 10 Sep 2026 23:59:32 +0200 Subject: [PATCH 4/9] bootstrap: the temporary control plane gets a temporary name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The substrate raises a control plane and a module will later declare one. If both are called `mesh-control` then for one moment two owners hold one container, and the host — which tracks what it owns — has no way to stop owning something without destroying it. That looked like a missing mechanism. It is a naming problem. The substrate's container becomes `temp-mesh-control` and the module's keeps the plain name: two containers, two owners, nothing to hand over. Dropping the temporary one from the bundle at the end is then destruction by omission, which is what the host already does to anything that leaves a declaration — and the right end for something named "temp" (novox/hq ADR 0067). The rename is textual and matches the QUOTED name, so the `mesh-control` inside the image reference is not caught by it. Read back afterwards: the produced bundle must call it the temporary name, and no other container may have been renamed. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/bootstrap/rewrite.go | 107 +++++++++++++++++++++++++++-- internal/bootstrap/rewrite_test.go | 98 ++++++++++++++++++++++++++ internal/bootstrap/verify_test.go | 10 +-- 3 files changed, 205 insertions(+), 10 deletions(-) diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index 2cdcde5..dc040e4 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -19,6 +19,28 @@ import ( // broker and no mesh. const ControlPlaneID = "control-plane" +// TempPrefix is what the substrate's control plane is renamed with. +// +// **This is the whole of how a carried resource becomes a declared one.** The substrate raises a +// control plane and a module later declares one, and for a moment both exist — which looked like a +// handover problem needing a way for the host to stop owning something without destroying it. It is +// not one. The temporary control plane is called `temp-mesh-control` and the permanent one is +// called `mesh-control`: two containers, two owners, nothing shared and nothing to hand over. At +// the end the temporary one is dropped from the bundle and the host removes it, which is exactly +// what should happen to something named "temp" (novox/hq ADR 0067). +// +// The name is also the audit. After the pivot, a machine running `mesh-control` and not +// `temp-mesh-control` has completed it; one running both stopped in the middle; one running only +// the temp has not started. That is readable from `docker ps` by somebody who knows nothing else. +const TempPrefix = "temp-" + +// ControlPlaneModule is the module the permanent control plane is installed as, and the name its +// container takes — the name the substrate's own control plane gives up here so that it can. +// +// Declared beside the rename rather than beside the step that uses it, because this is where the +// two names are decided together and where the reason for both of them is written down. +const ControlPlaneModule = "mesh-control" + // brokerAddressVar is what a token tells an enrolling node to dial. // // Not rewritten here — see the note in Rewrite — but reported, because it is the field most likely @@ -44,6 +66,12 @@ type Rewritten struct { // installer produced earlier. Changed bool + // WasCalled is what the template called the control plane's container; TempName is what the + // produced bundle calls it. Renamed is false when the template already used the temporary name. + WasCalled string + TempName string + Renamed bool + // Kept is every other container image, unchanged, as " ". Reported rather than // assumed: "postgres was left alone" is a claim, and this is the evidence for it. Kept []string @@ -56,8 +84,10 @@ type Rewritten struct { // Rewrite produces the bundle this machine will apply from the template it was given. // -// **One substitution, and it is textual.** The control plane's image becomes the id of the image -// this machine now holds; nothing else changes. Textual rather than parse-and-re-serialise because +// **Two substitutions, and both are textual.** The control plane's image becomes the id of the image +// this machine now holds, and its container is renamed `temp-mesh-control`; nothing else changes. +// The rename is what makes the pivot expressible at all — see TempPrefix. Textual rather than +// parse-and-re-serialise because // the produced file has to be *read* — a person getting a machine working must be able to open it, // see the substrate they recognise, and see exactly one thing different. Re-serialising a parsed // declaration would drop every comment in the template, and those comments are where the reasons @@ -118,6 +148,20 @@ func Rewrite(template []byte, imageID string) (Rewritten, error) { out.Changed = true } + // And the container is renamed, for the reason TempPrefix records. Done here rather than + // anywhere later because the bundle is the only place the name is decided: the apply creates + // the container from it, the verify asks that container questions, and the retirement takes + // this same resource back out. One name, one place, read by everything. + out.WasCalled, out.TempName = control.Name, TempPrefix+control.Name + if strings.HasPrefix(control.Name, TempPrefix) { + out.TempName = control.Name + } + renamed, err := renameContainer(out.Bundle, control.Name, out.TempName) + if err != nil { + return Rewritten{}, err + } + out.Bundle, out.Renamed = renamed, out.TempName != control.Name + // Read back, on the bytes that will actually be applied. Everything above is an intention // until the produced file is parsed and asked what it says. after, err := declaration.ParseFileTrusted(out.Bundle) @@ -135,27 +179,68 @@ func Rewrite(template []byte, imageID string) (Rewritten, error) { return Rewritten{}, fmt.Errorf( "the produced bundle still names the control plane %q, not %q", produced.Image, imageID) } + if produced.Name != out.TempName { + return Rewritten{}, fmt.Errorf( + "the produced bundle still calls the control plane's container %q, not %q. The "+ + "permanent one is a module and takes the plain name, so a substrate that kept it "+ + "would put two owners on one container", produced.Name, out.TempName) + } // And nothing else moved. A substitution on text can in principle catch more than it was // aimed at, and "postgres was left exactly as it was" is the claim this checks rather than - // asserts. - was := containerImages(before) + // asserts. Both the image AND the name, because there are now two substitutions. + wasImage, wasName := containerImages(before), containerNames(before) for id, image := range containerImages(after) { if id == ControlPlaneID { continue } - if was[id] != image { + if wasImage[id] != image { return Rewritten{}, fmt.Errorf( "rewriting the control plane's image also changed %q, from %q to %q. Only the "+ "mesh's own image may move; everything else is somebody else's image at "+ - "somebody else's registry", id, was[id], image) + "somebody else's registry", id, wasImage[id], image) } out.Kept = append(out.Kept, fmt.Sprintf("%s %s", id, image)) } + for id, name := range containerNames(after) { + if id == ControlPlaneID || wasName[id] == name { + continue + } + return Rewritten{}, fmt.Errorf( + "renaming the control plane's container also renamed %q, from %q to %q. Only the "+ + "control plane moves out of the way; every other container keeps the name the "+ + "substrate gave it", id, wasName[id], name) + } sortStrings(out.Kept) return out, nil } +// renameContainer changes one container's name in the bundle's text. +// +// **The quoted name, not the bare word.** `mesh-control` also appears inside the image reference +// the template carries (`…/mesh-control@sha256:…`) and could appear inside a command line; a bare +// substitution would catch those too. What is wanted is a JSON string that IS the name, so the +// quotes are part of what is matched — `"mesh-control"` matches the container's `name` and an +// action's `in`, which are exactly the places the name means the container, and nothing else. +// +// It refuses when the text does not contain what the parse says is there, for the same reason the +// image substitution does: the two would then be reading different things, and a rename that +// replaced nothing and reported success would leave the module and the substrate fighting over one +// container three steps later. +func renameContainer(bundle []byte, from, to string) ([]byte, error) { + if from == to { + return bundle, nil + } + quoted := []byte(`"` + from + `"`) + if bytes.Count(bundle, quoted) == 0 { + return nil, fmt.Errorf( + "the control plane's container is called %q according to the parsed template, and %s "+ + "is not in the file. Nothing was renamed, and the substrate would raise a "+ + "container the module also wants", from, quoted) + } + return bytes.ReplaceAll(bundle, quoted, []byte(`"`+to+`"`)), nil +} + // controlPlaneIn finds the container this installer replaces the image of. func controlPlaneIn(d *declaration.Declaration) (*declaration.Container, error) { for _, r := range d.Resources { @@ -186,6 +271,16 @@ func containerImages(d *declaration.Declaration) map[string]string { return images } +func containerNames(d *declaration.Declaration) map[string]string { + names := map[string]string{} + for _, r := range d.Resources { + if container, ok := r.(*declaration.Container); ok { + names[container.ID] = container.Name + } + } + return names +} + func identities(d *declaration.Declaration) []string { var ids []string for _, r := range d.Resources { diff --git a/internal/bootstrap/rewrite_test.go b/internal/bootstrap/rewrite_test.go index 11798ae..7902f1c 100644 --- a/internal/bootstrap/rewrite_test.go +++ b/internal/bootstrap/rewrite_test.go @@ -4,6 +4,8 @@ import ( "os" "strings" "testing" + + "github.com/novox/mesh-host/internal/declaration" ) // Each test names the decision it defends (novox/hq ADR 0017). @@ -221,3 +223,99 @@ func TestTheAddressNodesWillDialIsReportedAndNotRewritten(t *testing.T) { "knows this machine's address", out.BrokerAddress) } } + +// --------------------------------------------------------------------------------------------- +// The rename, which is what makes genesis a pivot rather than a handover (novox/hq ADR 0067). +// --------------------------------------------------------------------------------------------- + +// **This is the test that dissolves the blocker.** The substrate raises a control plane and a +// module later declares one; if both are called `mesh-control` then for one moment two owners hold +// one container, and the host — which tracks what it owns — has no way to stop owning something +// without destroying it. Nothing here invents such a mechanism. The substrate's container is +// called `temp-mesh-control` instead, and there are simply two containers. +func TestTheSubstratesControlPlaneMovesOutOfTheModulesWay(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if !out.Renamed { + t.Error("the rewrite reported nothing renamed, and the template named it mesh-control") + } + if out.TempName != "temp-mesh-control" { + t.Errorf("the substrate's control plane is called %q", out.TempName) + } + control, err := controlPlaneIn(out.Declaration) + if err != nil { + t.Fatal(err) + } + if control.Name != out.TempName { + t.Errorf("the produced bundle calls it %q, want %q", control.Name, out.TempName) + } + // And the plain name is free, which is the whole point: it belongs to the module now. + for _, name := range containerNames(out.Declaration) { + if name == ControlPlaneModule { + t.Errorf("the produced bundle still declares a container called %q, which the module "+ + "will also declare", ControlPlaneModule) + } + } +} + +// The image reference contains the string `mesh-control` too, and it is not a container name. A +// substitution that caught it would produce `…/temp-mesh-control@sha256:…`, which no registry +// serves — and it would be found inside a pull rather than here. +func TestTheImageReferenceIsNotMistakenForTheContainerName(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out.Bundle), TempPrefix+"mesh-control@") || + strings.Contains(string(out.Bundle), "/"+TempPrefix+"mesh-control") { + t.Error("the rename reached inside an image reference") + } +} + +// Everything else keeps the name the substrate gave it. The store and the broker are containers +// too, and a rename that moved them would leave a machine whose substrate the host cannot find. +func TestRenamingTheControlPlaneLeavesEveryOtherContainerAlone(t *testing.T) { + before, err := declaration.ParseFileTrusted(theRealBundle(t)) + if err != nil { + t.Fatal(err) + } + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + was, now := containerNames(before), containerNames(out.Declaration) + for id, name := range was { + if id == ControlPlaneID { + continue + } + if now[id] != name { + t.Errorf("%s was renamed from %q to %q", id, name, now[id]) + } + } +} + +// A re-run against a bundle this installer produced renames nothing and says so. The installer is +// run over and over while somebody gets a machine working, and a step that could not tell "already +// done" from "just done" makes the second run indistinguishable from the first. +func TestRewritingABundleThisAlreadyProducedRenamesNothing(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + second, err := Rewrite(first.Bundle, held) + if err != nil { + t.Fatal(err) + } + if second.Renamed { + t.Error("a bundle already naming temp-mesh-control was renamed again") + } + if second.TempName != first.TempName { + t.Errorf("the second pass calls it %q and the first called it %q", + second.TempName, first.TempName) + } + if string(second.Bundle) != string(first.Bundle) { + t.Error("rewriting a produced bundle changed it") + } +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go index 45eb857..3711d7d 100644 --- a/internal/bootstrap/verify_test.go +++ b/internal/bootstrap/verify_test.go @@ -85,9 +85,11 @@ func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) { if err != nil { t.Fatal(err) } - // Three long-running containers: the store, the broker and the control plane. The run-once and - // scheduled shapes are excluded on purpose — a step that has exited is not a fault. - want := []string{"mesh-store", "mesh-broker", "mesh-control"} + // Three long-running containers: the store, the broker and the TEMPORARY control plane. The + // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a + // fault. The name is `temp-mesh-control` because the permanent one is a module and takes the + // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. + want := []string{"mesh-store", "mesh-broker", "temp-mesh-control"} if len(verified.Running) != len(want) { t.Fatalf("confirmed %v running, want %v", verified.Running, want) } @@ -162,7 +164,7 @@ func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) { time.Second, 0, func(string) {}); err != nil { t.Fatal(err) } - if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + if !runtime.ran("docker exec " + TempPrefix + "mesh-control " + controlPlaneBinary + " status") { t.Errorf("the control plane was never asked anything: %v", runtime.commands) } } -- 2.54.0 From f534cf8b42e12b49d9dccf8efffca3f439ca8abe Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 10 Sep 2026 23:59:57 +0200 Subject: [PATCH 5/9] =?UTF-8?q?bootstrap:=20the=20rest=20of=20the=20pivot?= =?UTF-8?q?=20=E2=80=94=20enrol,=20registry,=20publish,=20reinstall,=20ret?= =?UTF-8?q?ire?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 6 to 10, which turn a substrate into a mesh that can maintain itself (novox/hq ADR 0067). 6 enrol a node record, a token, `mesh-host enrol`, and the host agent running. Proved by the mesh having HEARD from the node, not by a process existing: a host that cannot reach the broker looks exactly like a successful install until the first push applies nothing. 7 registry the module that gives this mesh an image store, registered from a --catalog checkout, assigned and pushed. Its image is upstream and never built (04-ISSUES/029) — a placeholder digest there is refused. Verified by asking `/v2/`, because a container that is up is not a registry that serves. 8 publish the carried image pushed into that registry, which assigns it the first manifest digest it has ever had. This is the hinge: without it the mesh works and can never upgrade itself. 9 control the control plane registered as an ordinary module pinned to that digest, with the substrate's own store connections delivered through `secret accept` — read out of the bundle that made them, because the mesh cannot invent a credential that predates it. 10 retire the temporary control plane dropped from the bundle and removed by the host's ordinary removal pass. Every step asks before it acts and reports "already done". No step leaves the machine without a control plane: steps 9 and 10 overlap deliberately, and two stateless control planes are untidy rather than broken. mesh-control's `internal/builder`.PublishImage is mirrored rather than imported — tier 0 depends on nothing that must be installed first — with one correction: the digest is chosen from RepoDigests by repository instead of taken as element zero, so an image pushed to two registries cannot silently pin this mesh to the wrong one. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- cmd/mesh-bootstrap/main.go | 105 ++++++++- cmd/mesh-bootstrap/main_test.go | 53 ++++- internal/bootstrap/bootstrap.go | 229 ++++++++++++++++--- internal/bootstrap/control.go | 335 ++++++++++++++++++++++++++++ internal/bootstrap/control_test.go | 213 ++++++++++++++++++ internal/bootstrap/enrol.go | 276 +++++++++++++++++++++++ internal/bootstrap/enrol_test.go | 223 ++++++++++++++++++ internal/bootstrap/module.go | 145 ++++++++++++ internal/bootstrap/publish.go | 173 ++++++++++++++ internal/bootstrap/publish_test.go | 197 ++++++++++++++++ internal/bootstrap/registry.go | 204 +++++++++++++++++ internal/bootstrap/registry_test.go | 216 ++++++++++++++++++ internal/bootstrap/retire.go | 291 ++++++++++++++++++++++++ internal/bootstrap/retire_test.go | 165 ++++++++++++++ internal/bootstrap/talk.go | 119 ++++++++++ 15 files changed, 2910 insertions(+), 34 deletions(-) create mode 100644 internal/bootstrap/control.go create mode 100644 internal/bootstrap/control_test.go create mode 100644 internal/bootstrap/enrol.go create mode 100644 internal/bootstrap/enrol_test.go create mode 100644 internal/bootstrap/module.go create mode 100644 internal/bootstrap/publish.go create mode 100644 internal/bootstrap/publish_test.go create mode 100644 internal/bootstrap/registry.go create mode 100644 internal/bootstrap/registry_test.go create mode 100644 internal/bootstrap/retire.go create mode 100644 internal/bootstrap/retire_test.go create mode 100644 internal/bootstrap/talk.go diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 894462c..633f668 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -8,8 +8,11 @@ // cannot be folded into it without making that sentence false. Same tier, same repository, // different program. // -// What it does not do is enrol this machine, register modules or assign them. It stops at a -// running substrate with a control plane that answers, which is a mesh of one node. +// Genesis is a pivot (novox/hq ADR 0067). It raises a substrate whose control plane is named by the +// digest of its own configuration — legal exactly where nothing could have served an image — then +// enrols this machine, installs the registry module, pushes that image into it to get the manifest +// digest it has never had, reinstalls the control plane as an ordinary module pinned to it, and +// drops the temporary one. Without --catalog it stops after the substrate and says why. package main import ( @@ -17,7 +20,9 @@ import ( "encoding/json" "flag" "fmt" + "io" "net" + "net/http" "os" "os/signal" "syscall" @@ -35,27 +40,56 @@ var version = "development build" const ( defaultTemplate = "substrate.lock" defaultOut = "/var/lib/mesh-host/substrate.lock" + defaultRegistry = "127.0.0.1:5000" + defaultHost = "/usr/local/bin/mesh-host" + defaultService = "mesh-host.service" ) const usage = `mesh-bootstrap — make a bare machine into a mesh - bootstrap preflight, load, bundle, apply, verify (the default) + bootstrap the ten steps below (the default) version + 1 preflight what has to be true before anything is changed + 2 load the control plane's image, carried in this installer + 3 bundle the substrate, named for this machine + 4 apply raise it + 5 verify it is up, and the control plane replies + 6 enrol this machine becomes the mesh's first node + 7 registry install the module that gives this mesh an image store + 8 publish push the control plane's image into it, for its first digest + 9 control reinstall the control plane as an ordinary module, pinned to that digest + 10 retire drop the temporary control plane; the host removes it + --bundle the substrate template to build this machine's bundle from (default ` + defaultTemplate + `) --out where the produced bundle is written, for a person to read (default ` + defaultOut + `) --state where this node records what it has applied (default ` + store.DefaultPath + `) + --catalog a checkout of the mesh's catalogue, holding the registry's and the + control plane's manifests. Without it this stops after step 5 + --node the name this machine is known by (default: its hostname) + --registry where this mesh keeps its own images (default ` + defaultRegistry + `) + every node pulls the control plane from this, so on a mesh of more + than one machine it must be an address the others can reach + --host the mesh-host binary on this machine (default ` + defaultHost + `) + --host-service the unit that supervises it (default ` + defaultService + `) + --host-in-background start the host unsupervised instead. It does not survive + a reboot. This is what a lab does and what no real machine should --system which operating system this is; by default it is asked --timeout how long any single probe may take (default 30s) --wait how long a thing that is merely starting is given (default 3m) --dry-run everything that does not change the machine --json machine-readable output -It carries the control plane's image and applies a substrate. Every step is idempotent: -run it again after fixing whatever it named, and the steps that already succeeded say so. +Genesis is a pivot: a temporary control plane installs the registry that makes it +permanent. The temporary one is called temp-mesh-control and the permanent one is +called mesh-control, so they are two containers with two owners and there is nothing +to hand over. + +Every step is idempotent: run it again after fixing whatever it named, and the steps +that already succeeded say so. ` func main() { @@ -85,6 +119,14 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { Template: defaultTemplate, Out: defaultOut, State: store.DefaultPath, + Registry: defaultRegistry, + Host: defaultHost, + // The machine's own name, because that is what a person already calls it and an installer + // inventing a different one would leave the mesh naming a machine nobody recognises. It is + // read here rather than inside the bootstrap so that --node overrides a fact rather than a + // default computed halfway through. + Node: hostname(), + HostService: defaultService, // Longer than the host's 10s: these probes reach a container runtime that may be busy // pulling, and a probe that times out on a working machine is a false refusal. Timeout: 30 * time.Second, @@ -131,6 +173,14 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from") set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written") set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied") + set.StringVar(&opts.Catalogue, "catalog", opts.Catalogue, + "a checkout of the mesh's catalogue; without it this stops after the substrate") + set.StringVar(&opts.Node, "node", opts.Node, "the name this machine is known by") + set.StringVar(&opts.Registry, "registry", opts.Registry, "where this mesh keeps its own images") + set.StringVar(&opts.Host, "host", opts.Host, "the mesh-host binary on this machine") + set.StringVar(&opts.HostService, "host-service", opts.HostService, "the unit that supervises it") + set.BoolVar(&opts.HostInBackground, "host-in-background", false, + "start the host unsupervised; it does not survive a reboot") set.StringVar(&opts.System, "system", opts.System, "which operating system this is") set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take") set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given") @@ -148,8 +198,9 @@ func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bo } } result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{ - Run: apply.ExecRunner, - Dial: dial, + Run: apply.ExecRunner, + Dial: dial, + Fetch: fetch, }, say) // Printed whichever way it went. What the installer got through before it stopped is on @@ -177,6 +228,46 @@ func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bo } } +// hostname is what this machine calls itself, or empty. +// +// Empty rather than a guess: a machine that cannot say its own name is one the installer must be +// told about, and `mesh-bootstrap-0` would be a name in the mesh's records that matches nothing +// anybody types anywhere else. The refusal happens at step 6, where the name is first needed. +func hostname() string { + name, err := os.Hostname() + if err != nil { + return "" + } + return name +} + +// fetch asks an HTTP endpoint and reports what it said. +// +// Plain HTTP, and only at the mesh's own registry: it is reached over the mesh's private network, +// which is already the encrypted and authenticated thing, and a second layer inside it would be +// certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder` pushes to it on the same reasoning). +// +// The body is read with a limit. What is asked for is a status and a short JSON answer, and a +// registry that answered with a gigabyte would otherwise be an installer that never returns. +func fetch(ctx context.Context, url string) (int, string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, "", err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, "", err + } + defer response.Body.Close() + + said, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + return response.StatusCode, "", err + } + return response.StatusCode, string(said), nil +} + // dial answers whether a TCP address responds. // // A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go index 6731428..95815d1 100644 --- a/cmd/mesh-bootstrap/main_test.go +++ b/cmd/mesh-bootstrap/main_test.go @@ -105,9 +105,60 @@ func TestTheUsageTextAndTheFlagsAgree(t *testing.T) { t.Errorf("--%s exists and the usage text does not mention it", name) } } - for _, promised := range []string{"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json"} { + for _, promised := range []string{ + "bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json", + "catalog", "node", "registry", "host", "host-service", "host-in-background", + } { if !declared[promised] { t.Errorf("the usage text promises --%s and no such flag exists", promised) } } } + +// The pivot's defaults are the documented ones too, and the one that has no default is the one +// that decides whether the pivot happens at all. +func TestThePivotsDefaultsAreTheDocumentedOnes(t *testing.T) { + _, opts, _, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Registry != defaultRegistry { + t.Errorf("default registry is %q; the usage text says %q", opts.Registry, defaultRegistry) + } + if opts.Host != defaultHost { + t.Errorf("default host binary is %q; the usage text says %q", opts.Host, defaultHost) + } + if opts.HostService != defaultService { + t.Errorf("default host service is %q; the usage text says %q", + opts.HostService, defaultService) + } + if opts.HostInBackground { + t.Error("the host is started unsupervised by default, and no real machine should") + } + // **No default, deliberately.** A catalogue this installer went looking for on its own would + // be a checkout somebody else made, at whatever commit they left it on — and it decides which + // image the mesh's control plane is pinned to for ever after. + if opts.Catalogue != "" { + t.Errorf("--catalog defaults to %q; without one the installer stops at the substrate", + opts.Catalogue) + } + // The machine's own name, because that is what a person already calls it. + if opts.Node == "" { + t.Error("no default node name; this machine can say what it is called") + } +} + +// --node overrides the machine's own name rather than being ignored because a default was already +// computed. The name is what the mesh's records, tokens, assignments and pushes all name. +func TestTheNodeNameCanBeSaid(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--node", "anchor", "--catalog", "/somewhere"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Node != "anchor" { + t.Errorf("--node anchor parsed as %q", opts.Node) + } + if opts.Catalogue != "/somewhere" { + t.Errorf("--catalog parsed as %q", opts.Catalogue) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 44163b1..1771aee 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -25,6 +25,7 @@ package bootstrap import ( "context" "fmt" + "strings" "time" ) @@ -33,15 +34,28 @@ import ( type Step string const ( - StepPreflight Step = "preflight" - StepLoad Step = "load" - StepBundle Step = "bundle" - StepApply Step = "apply" - StepVerify Step = "verify" + StepPreflight Step = "preflight" + StepLoad Step = "load" + StepBundle Step = "bundle" + StepApply Step = "apply" + StepVerify Step = "verify" + StepEnrol Step = "enrol" + StepRegistry Step = "registry" + StepPublish Step = "publish" + StepControlPlane Step = "control-plane" + StepRetire Step = "retire" ) -// Steps in the order they happen, so a failure can say "step 2 of 5". -var Steps = []Step{StepPreflight, StepLoad, StepBundle, StepApply, StepVerify} +// Steps in the order they happen, so a failure can say "step 2 of 10". +// +// The first five make a machine; the last five make a mesh that can maintain itself. They are one +// program because they are one procedure — the whole reason the pivot exists is that steps 7 to 9 +// cannot happen without steps 1 to 5, and steps 1 to 5 leave something that cannot be upgraded +// without steps 7 to 9 (novox/hq ADR 0067). +var Steps = []Step{ + StepPreflight, StepLoad, StepBundle, StepApply, StepVerify, + StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, +} // Error is a failure, named by the step it happened in. type Error struct { @@ -86,8 +100,37 @@ type Options struct { // Wait is how long something that is merely starting is given: a socket-activated container // runtime, a control plane opening its stores. Wait time.Duration + + // Node is the name this machine is known by in the mesh. Everything after the substrate names + // it: the record, the token, the assignment, the push. + Node string + + // Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and + // the control plane's manifests are read from. Empty stops the installer after the substrate: + // there is no pivot without manifests, and pretending otherwise would leave a machine that + // looks installed and cannot upgrade itself. + Catalogue string + + // Registry is where this mesh's own images live, as this machine reaches it. Every node will + // pull the control plane from what this says, so on a mesh of more than one machine it must be + // an address the others can reach. + Registry string + + // Host is the `mesh-host` binary on this machine — the program that enrols and then holds the + // machine to what the mesh says. The installer runs it; it does not contain it. + Host string + // HostService is the unit that supervises it. Started and enabled, never written: what a unit + // says is a packaging decision, and an installer inventing one would put a file on the machine + // that whatever installed the host will disagree with. + HostService string + // HostInBackground starts the host unsupervised instead, which is what the lab does and what no + // real machine should do — it does not survive a reboot. + HostInBackground bool } +// pivots reports whether this run goes past the substrate. +func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" } + // Deps are the ways this program reaches outside itself. Injected so the whole of it can be // tested without a container runtime, a network, or a machine to break — the same reason // `internal/apply` takes a Runner (novox/hq ADR 0017). @@ -97,6 +140,10 @@ type Deps struct { // Dial reports whether a TCP address answers, for "can this machine reach the registries the // bundle names". Dial func(ctx context.Context, address string) error + // Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own + // registry: a container that is up is not a registry that serves, and `/v2/` is the one + // question whose answer means it is. + Fetch func(ctx context.Context, url string) (int, string, error) } // Result is what the bootstrap did, in the shape `--json` prints. @@ -123,10 +170,34 @@ type Result struct { // Running is the substrate's containers, confirmed up. Running []string `json:"running,omitempty"` - // Answered is what the control plane said back — not merely that it is up. - Answered string `json:"control-plane,omitempty"` + // Answered is what the temporary control plane said back — not merely that it is up. + Answered string `json:"temporary-control-plane,omitempty"` + // Temporary is what the substrate's control plane is called, which is not what the module's is. + Temporary string `json:"temporary-container,omitempty"` - // Stopped names why a dry run went no further. Empty on a real run. + // Node is this machine's name in the mesh, and how it came to be enrolled and heard from. + Node string `json:"node,omitempty"` + NodeAdded bool `json:"node-record-created,omitempty"` + Enrolled bool `json:"enrolled-now,omitempty"` + Agent string `json:"host-agent,omitempty"` + + // Registry is the mesh's own artifact store, once it answers. + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + + // Permanent is the control plane as an ordinary module. + Permanent string `json:"permanent-container,omitempty"` + PermanentAnswered string `json:"permanent-control-plane,omitempty"` + StoresDelivered []string `json:"stores-delivered,omitempty"` + TemporaryRetired bool `json:"temporary-retired,omitempty"` + TemporaryWasGone bool `json:"temporary-was-already-gone,omitempty"` + TemporaryRemovedAt int `json:"resources-removed,omitempty"` + + // Stopped names why a run went no further. Empty on a run that pivoted. Stopped string `json:"stopped,omitempty"` } @@ -141,9 +212,41 @@ type Result struct { // answer to it is to run this again: re-running is the retry, and it is one a person chooses after // reading which step failed and why. // -// **What this does NOT do: enrolment, the module catalogue, and assignment.** It stops at a running -// substrate with a control plane that replies — a mesh of one node with nothing joined to it. See -// the marker at the end. +// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is +// named by the digest of its own configuration, because nothing has ever served that image and +// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine +// enrols, the registry module is installed, the carried image is pushed INTO that registry — which +// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary +// module pinned to it. The temporary one is then dropped from the bundle and the host removes it. +// +// **What makes the last part expressible is a name.** The substrate's control plane is called +// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners: +// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction +// by omission is the right end for something named "temp". +// +// Without --catalog it stops after step 5 and says so, because there are no manifests to install +// and a machine that looks installed and cannot upgrade itself is worse than one that stopped. +// +// **What an interruption leaves, at every step, and how a re-run continues.** This matters more +// here than anywhere else in the repository, because a machine left without a control plane cannot +// be fixed remotely — so no step may leave one: +// +// 1–3 nothing on the machine but a written file. Re-run: the bundle is produced again. +// 4 a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest. +// 5 everything up; something did not answer yet. Re-run: it is asked again. +// 6 a node record and possibly a spent token. Re-run: `node list` finds the record, the +// identity file says whether this machine enrolled, and a fresh token is issued if not. +// 7 the registry registered, assigned, maybe not applied. Re-run: registered again (an +// upsert), pushed again, waited for again. The temporary control plane is untouched. +// 8 the image pushed and the digest unread. Re-run: the registry is asked what it holds and +// the answer is the same digest; nothing is pushed twice. +// 9 the module registered and the container not yet up, OR up beside the temporary one. Both +// are working states: the mesh has a control plane throughout. Re-run: continues. +// 10 the bundle rewritten and the container still there. Re-run: the bundle already omits it +// and the apply removes it; a bundle that already omits it reports "already dropped". +// +// The only step that cannot be undone by re-running is enrolment, and that is refused rather than +// repeated: a second identity is one the mesh does not know, and the mesh believes the first. func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) { if say == nil { say = func(string) {} @@ -181,12 +284,21 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepBundle, err) } result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out + result.Temporary = rewritten.TempName + if rewritten.Renamed { + say(fmt.Sprintf(" control plane %s, renamed from %s", + rewritten.TempName, rewritten.WasCalled)) + say(" the permanent one is a module and takes the plain name; " + + "this one is dropped at the end") + } else { + say(" control plane " + rewritten.TempName + " — the template already named it that") + } if rewritten.Changed { - say(fmt.Sprintf(" control plane %s", rewritten.Now)) + say(fmt.Sprintf(" its image %s", rewritten.Now)) say(fmt.Sprintf(" replacing %s, named in %d place(s)", rewritten.Was, rewritten.Places)) } else { - say(fmt.Sprintf(" control plane %s — the template already named it, nothing rewritten", + say(fmt.Sprintf(" its image %s — the template already named it, nothing rewritten", rewritten.Now)) } for _, kept := range rewritten.Kept { @@ -207,6 +319,15 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // loading, applying and asking the result questions, and none of those can be answered by // not doing them. say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources)) + if o.pivots() { + // Named rather than attempted. Everything from step 6 on is a conversation with a + // control plane that a dry run has not raised, so there is nothing to ask and nothing + // honest to report about the answers. + say(" would then enrol " + o.Node + ", install the registry from " + + o.Catalogue + ", push the control plane's image into it,") + say(" reinstall the control plane as a module, and drop " + + rewritten.TempName) + } result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + "loaded or applied" say("\n" + result.Stopped) @@ -242,19 +363,75 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepVerify, err) } - say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + if !o.pivots() { + // Stopped, and said plainly. What has been raised works and cannot be upgraded: its + // control plane is named by an image id, which no registry serves, so nothing can ever + // replace it with a newer one. That is the whole of what the pivot fixes, and it needs + // manifests, and manifests come from a checkout somebody has to point this at. + result.Stopped = "no --catalog was given, so this stopped at the substrate. " + + "The control plane is named by the digest of its own configuration and no registry " + + "serves it, so this mesh cannot yet upgrade itself. Run again with " + + "--catalog to finish the pivot; every step " + + "above will say it is already done" + say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + say(result.Stopped) + return result, nil + } - // NEXT STAGE — NOT IMPLEMENTED HERE. + // ---- 6. enrol ------------------------------------------------------------------------- // - // What remains between "a mesh exists" and "a mesh does something": issuing this machine a - // token and enrolling it as its own first node, registering the module catalogue with the - // control plane, and assigning modules to nodes. All three are conversations with the control - // plane that has just been proved to reply, so they belong after this point and inside none of - // the steps above. - // - // Left out rather than half-written. Everything above changes a machine; all of that changes a - // mesh, and a program that did both would have two jobs and one name. - say("not done here: enrolment, the module catalogue, and assignment.") + // From here on the mesh is being told things, and the way to tell it anything is to run its + // own binary inside its own container. `temporary` is the substrate's control plane; the + // module's is a different container with a different name and does not exist yet. + temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout} + + say("enrol — this machine joins the mesh it is running") + enrolled, err := Enrol(ctx, o, sys, temporary, say) + result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined + result.Agent = enrolled.Agent + if err != nil { + return result, failed(StepEnrol, err) + } + + // ---- 7. registry ---------------------------------------------------------------------- + say("registry — somewhere for this mesh to keep its own images") + registry, err := InstallRegistry(ctx, o, d, temporary, say) + result.Registry, result.RegistryRunning = registry.Address, registry.Container + result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered + if err != nil { + return result, failed(StepRegistry, err) + } + + // ---- 8. publish ----------------------------------------------------------------------- + say("publish — the control plane's image gets its first manifest digest") + published, err := PublishControlPlane(ctx, o, d, loaded.ID, say) + result.PublishedAs, result.PublishedAlready = published.Reference, published.Already + if err != nil { + return result, failed(StepPublish, err) + } + + // ---- 9. control plane ----------------------------------------------------------------- + say("control plane — installed as an ordinary module, pinned to that digest") + permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration, + published.Reference, say) + result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered + result.StoresDelivered = permanent.Delivered + if err != nil { + return result, failed(StepControlPlane, err) + } + + // ---- 10. retire ----------------------------------------------------------------------- + say("retire — the temporary control plane is dropped from the bundle") + retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say) + result.TemporaryRetired = retired.Gone || retired.Already + result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed + if err != nil { + return result, failed(StepRetire, err) + } + + say("\nthis machine is a mesh of one node, and the control plane it runs is a module " + + "pinned to an image its own registry serves.") + say("what remains is somebody else's: adding nodes, and assigning what they should run.") return result, nil } diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go new file mode 100644 index 0000000..c406b53 --- /dev/null +++ b/internal/bootstrap/control.go @@ -0,0 +1,335 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the +// environment. +// +// `MESH_STORE_` is what the control plane reads (mesh-control's `internal/store`.Variable) +// and putting a password in a container's environment puts it in `docker inspect` for ever. So a +// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value +// into that file on the machine, and nothing but the process reads it. +const storeFileSuffix = "_FILE" + +// storeVariablePrefix is the front of the same names. +const storeVariablePrefix = "MESH_STORE_" + +// Permanent is what step 9 did. +type Permanent struct { + Installed + // Container is what the module calls its container, confirmed running. + Container string + // Image is what it is pinned to — the digest step 8's push produced. + Image string + // Delivered is every store connection accepted into it, by secret name. + Delivered []string + // Answered is what the permanent control plane said back. + Answered string +} + +// InstallControlPlane makes the control plane an ordinary module. +// +// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary +// control plane composes a declaration naming the registry-pinned image, publishes it, and this +// node's host creates the container. Nothing is asked to replace itself while running, which is +// what makes the whole thing expressible: the container being created is called `mesh-control` and +// the one composing it is called `temp-mesh-control`, so there are two of them and neither is in +// the other's way. +// +// **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.** +// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are +// the credentials the substrate bundle created the databases with. Generating replacements would +// put thirty-two random bytes where a working connection string has to be, and the control plane +// would come up unable to open a single context. So they go in through `secret accept`, which is +// exactly the path for a value the mesh must carry and could not have invented — and they are read +// out of the bundle this installer produced rather than reconstructed, because the bundle is what +// created them and a second opinion about what a DSN should say is a second chance to be wrong. +func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane, + substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) { + + out := Permanent{Image: image} + + manifest, err := readManifest(o.Catalogue, ControlPlaneModule) + if err != nil { + return out, fmt.Errorf( + "%w\n"+ + "This is the manifest that makes the control plane an ordinary module. Without it "+ + "the machine keeps the temporary control plane the substrate raised, which works "+ + "and cannot be upgraded — so the install stops here rather than pretending to "+ + "have pivoted", err) + } + + pinned, places, err := pinImage(manifest, image) + if err != nil { + return out, err + } + say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places)) + + container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned)) + if err != nil { + return out, err + } + out.Container = container + if container == "" { + return out, fmt.Errorf( + "the %s module's container has no name, so nothing can be verified afterwards", + ControlPlaneModule) + } + + installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say) + out.Installed = installed + if err != nil { + return out, err + } + + // The connections, before the push that would otherwise deliver random bytes for them. + delivered, err := deliverStores(ctx, o, control, pinned, substrate, say) + out.Delivered = delivered + if err != nil { + return out, err + } + + if out.Pushed, err = pushNode(ctx, o, control, say); err != nil { + return out, err + } + + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + // And it answers, which is the same question step 5 asked of the temporary one and for the + // same reason: `status` opens all three stores, so a reply proves the sealed connections it + // was given are the ones the substrate made. Asked of the NEW container — this is the only + // moment in the program where two control planes are running, and asking the wrong one would + // report the temporary one's health as the permanent one's. + answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say) + if err != nil { + return out, err + } + out.Answered = answered + return out, nil +} + +// pinImage replaces the catalogue's placeholder digest with what the registry assigned. +// +// **Textual, and every place it appears.** A manifest may name its image in more than one resource +// — the catalogue's converted modules routinely carry a runtime container beside the application's +// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves +// something pointing at an image nothing serves, and it fails half way through an apply rather +// than here. +// +// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already +// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a +// control plane that is not the image this machine just published — which is the one thing this +// step exists to guarantee. +func pinImage(manifest []byte, reference string) ([]byte, int, error) { + places := bytes.Count(manifest, []byte(placeholderDigest)) + if places == 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+ + "pin to the image this machine just published.\n"+ + "A manifest already naming a digest was pinned by somebody else, to some other "+ + "build. Registering it would install a control plane that is not the one this "+ + "installer carried and pushed", ControlPlaneModule, placeholderDigest) + } + // The reference the registry gave back is `/@sha256:…`, and what the + // manifest holds is `@sha256:0…0`. Replacing only the digest would leave the + // manifest's own repository name in front of it — which may be `mesh-control` with no + // registry, and a runtime would then pull it from the internet. The whole reference moves. + var out bytes.Buffer + rest := manifest + for { + at := bytes.Index(rest, []byte(placeholderDigest)) + if at < 0 { + out.Write(rest) + break + } + // Back up over the repository this digest belongs to, which runs to the opening quote. + start := bytes.LastIndexByte(rest[:at], '"') + if start < 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest has a placeholder digest that is not inside a JSON "+ + "string, so the installer cannot tell what image it belongs to", ControlPlaneModule) + } + out.Write(rest[:start+1]) + out.WriteString(reference) + rest = rest[at+len(placeholderDigest):] + } + pinned := out.Bytes() + + // Read back. A substitution on text can catch more than it was aimed at, and the manifest is + // about to be handed to the mesh as the description of what it runs. + var checked map[string]any + if err := json.Unmarshal(pinned, &checked); err != nil { + return nil, 0, fmt.Errorf( + "pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err) + } + if bytes.Contains(pinned, []byte(placeholderDigest)) { + return nil, 0, fmt.Errorf( + "the %s module's manifest still carries a placeholder digest after pinning", + ControlPlaneModule) + } + return pinned, places, nil +} + +// controlPlaneResourceIn is the id of the resource that runs the control plane. +// +// The manifest is written by the catalogue and the installer does not get to name its resources. +// What it can do is find the one container whose image is the one just pinned — and when a manifest +// declares exactly one container, that is the answer without any searching at all. +func controlPlaneResourceIn(manifest []byte) string { + var m struct { + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "" + } + var containers []string + for _, r := range m.Resources { + if r.Type == "container" { + containers = append(containers, r.ID) + } + } + if len(containers) == 1 { + return containers[0] + } + // More than one, so the name has to be guessed at rather than derived — and the catalogue's + // own convention for the resource that IS the module is `container`, with anything else beside + // it named for what it does. + for _, id := range containers { + if id == "container" || id == ControlPlaneModule || id == "control-plane" { + return id + } + } + return "" +} + +// deliverStores carries the substrate's own database connections into the module. +// +// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls +// these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a +// path, and the module's own-secret that writes that path is the secret to accept the connection +// as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the +// secret is called `inventory`, would seal a connection string under a name nothing reads and +// leave the mesh to invent random bytes for the one that is. +func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, + substrate *declaration.Declaration, say func(string)) ([]string, error) { + + wanted, err := storeSecretsIn(manifest) + if err != nil { + return nil, err + } + if len(wanted) == 0 { + return nil, fmt.Errorf( + "the %s module's manifest asks for no store connections. A control plane reaches each "+ + "context through its own credential (novox/hq ADR 0008), so a manifest naming none "+ + "describes a control plane that can open nothing.\n"+ + "The shape this installer delivers into is a file per context, named by an "+ + "own-secret, with %s%s in the container's environment pointing at it", + ControlPlaneModule, storeVariablePrefix, storeFileSuffix) + } + + temporary, err := controlPlaneIn(substrate) + if err != nil { + return nil, err + } + + var delivered []string + for _, context := range sortedKeys(wanted) { + secret := wanted[context] + connection := temporary.Env[storeVariablePrefix+context] + if strings.TrimSpace(connection) == "" { + return delivered, fmt.Errorf( + "the %s module wants the %s store's connection and the bundle this installer "+ + "produced does not name one: its control plane has no %s.\n"+ + "These connections are the substrate's, created at genesis — the mesh cannot "+ + "invent them and the installer will not guess at one", + ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context) + } + + // Into the container as a file, because `secret accept` reads a file or a prompt and the + // installer has neither a terminal to be prompted at nor a way to write to a command's + // standard input through the runner every applier in this repository shares. + at := "/accepting-" + strings.ToLower(context) + if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context), + []byte(connection), at); err != nil { + return delivered, err + } + if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, + "--from", at); err != nil { + return delivered, err + } + delivered = append(delivered, secret) + say(" accepted " + secret + " — the " + strings.ToLower(context) + + " store, as the substrate made it") + } + return delivered, nil +} + +// storeSecretsIn pairs each context with the secret its connection must be accepted as. +// +// Read out of the manifest twice over: the container's environment says which contexts are wanted +// and what file each expects, and the module's own-secrets say which secret writes which file. A +// pair that does not meet is refused rather than half-delivered. +func storeSecretsIn(manifest []byte) (map[string]string, error) { + var m struct { + OwnSecrets map[string]string `json:"own-secrets"` + Resources []struct { + Type string `json:"type"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) + } + + byPath := map[string]string{} + for name, path := range m.OwnSecrets { + byPath[path] = name + } + + wanted := map[string]string{} + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + for key, path := range r.Env { + if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) { + continue + } + context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix) + secret, ok := byPath[path] + if !ok { + return nil, fmt.Errorf( + "the %s module's container reads the %s store's connection from %s, and no "+ + "own-secret of that module writes that file.\n"+ + "So the mesh would seal nothing there and the control plane would find an "+ + "empty file where a connection string has to be. The manifest has to name "+ + "the two ends the same", + ControlPlaneModule, strings.ToLower(context), path) + } + wanted[context] = secret + } + } + return wanted, nil +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go new file mode 100644 index 0000000..bf4d97c --- /dev/null +++ b/internal/bootstrap/control_test.go @@ -0,0 +1,213 @@ +package bootstrap + +import ( + "context" + "strings" + "testing" + "time" +) + +// Step 9 is where the control plane stops being a special case. These tests defend the two things +// that could go wrong quietly: pinning it to the wrong image, and delivering it store connections +// the mesh invented rather than the ones the substrate actually made. + +// theControlPlaneModule is the shape this installer codes against: one container using the +// catalogue's placeholder-digest convention, with each store connection delivered as a sealed +// secret written into a file and MESH_STORE__FILE pointing at it. +const theControlPlaneModule = `{ + "module": "mesh-control", + "version": "1", + "capabilities": ["container-runtime"], + "own-secrets": { + "inventory-store": "/var/lib/mesh/control/inventory", + "identity-store": "/var/lib/mesh/control/identity", + "licences-store": "/var/lib/mesh/control/licences" + }, + "resources": [ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + {"id": "container", "type": "container", "name": "mesh-control", + "image": "mesh-control@` + placeholderDigest + `", + "network": "host", "args": ["serve"], + "env": { + "MESH_STORE_INVENTORY_FILE": "/var/lib/mesh/control/inventory", + "MESH_STORE_IDENTITY_FILE": "/var/lib/mesh/control/identity", + "MESH_STORE_LICENCES_FILE": "/var/lib/mesh/control/licences" + }} + ] +}` + +const pushedReference = "127.0.0.1:5000/mesh-control@sha256:" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + +// **The whole reference moves, not only the digest.** The manifest's placeholder names a +// repository too, and replacing sixty-four zeros inside it would leave `mesh-control@sha256:…` +// with no registry in front — which a runtime would go to the internet for, and this mesh's +// control plane exists in no public registry by design. +func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) { + pinned, places, err := pinImage([]byte(theControlPlaneModule), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 1 { + t.Errorf("the placeholder was found in %d place(s)", places) + } + if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) { + t.Errorf("the manifest does not name the pushed image:\n%s", pinned) + } + if strings.Contains(string(pinned), `"mesh-control@sha256:`) { + t.Errorf("the digest was replaced and the manifest's own repository name was left in "+ + "front of it, so nothing says which registry serves it:\n%s", pinned) + } +} + +// A manifest already naming a real digest was pinned by somebody else, to some other build. +// Registering it would install a control plane that is not the image this machine just published, +// which is the one thing this step exists to guarantee. +func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) { + already := strings.Replace(theControlPlaneModule, placeholderDigest, + "sha256:"+strings.Repeat("9", 64), 1) + if _, _, err := pinImage([]byte(already), pushedReference); err == nil { + t.Fatal("a manifest already pinned to some other image was accepted") + } +} + +// Every placeholder moves. A manifest naming its image in a second resource — a runtime container +// beside the application's, which the catalogue's converted modules routinely carry — would +// otherwise be left half pinned, and fail inside an apply rather than here. +func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { + twice := strings.Replace(theControlPlaneModule, + `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"},`, + `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + {"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true, + "image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1) + + pinned, places, err := pinImage([]byte(twice), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 2 { + t.Errorf("the placeholder was found in %d place(s), and the manifest names it twice", places) + } + if strings.Contains(string(pinned), placeholderDigest) { + t.Error("a placeholder survived the pinning") + } +} + +// **The connections are the substrate's, and they are read out of the bundle that made them.** +// The mesh cannot invent them: they are the credentials the substrate created the databases with, +// and thirty-two random bytes in their place would leave the control plane unable to open a single +// context. The pairing is read from the manifest so that whatever the catalogue calls these +// secrets is what is delivered. +func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { + wanted, err := storeSecretsIn([]byte(theControlPlaneModule)) + if err != nil { + t.Fatal(err) + } + for context, secret := range map[string]string{ + "INVENTORY": "inventory-store", + "IDENTITY": "identity-store", + "LICENCES": "licences-store", + } { + if wanted[context] != secret { + t.Errorf("the %s store's connection would be accepted as %q, want %q", + context, wanted[context], secret) + } + } + + // And the values are the substrate's own, taken from the produced bundle rather than composed. + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(theControlPlaneModule), rewritten.Declaration, func(string) {}) + if err != nil { + t.Fatal(err) + } + if len(delivered) != 3 { + t.Fatalf("%d connections were delivered, and the mesh holds three contexts: %v", + len(delivered), delivered) + } + for _, secret := range delivered { + if !runtime.ran("secret accept anchor mesh-control " + secret + " --from") { + t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands) + } + } +} + +// A manifest whose container reads a file no own-secret writes is refused. The mesh would seal +// nothing there and the control plane would find an empty file where a connection string has to +// be — which presents as a control plane that will not start, three steps from the cause. +func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { + mismatched := strings.Replace(theControlPlaneModule, + `"inventory-store": "/var/lib/mesh/control/inventory"`, + `"inventory-store": "/var/lib/mesh/control/somewhere-else"`, 1) + + _, err := storeSecretsIn([]byte(mismatched)) + if err == nil { + t.Fatal("a manifest whose two ends do not meet was accepted") + } + if !strings.Contains(err.Error(), "own-secret") { + t.Errorf("the refusal does not say which half is missing: %v", err) + } +} + +// A manifest asking for no store connections at all describes a control plane that can open +// nothing, and the refusal says what shape the installer delivers into — because the manifest is +// written in another repository and this is where the two have to agree. +func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + bare := `{"module":"mesh-control","version":"1","resources":[ + {"id":"container","type":"container","name":"mesh-control", + "image":"mesh-control@` + placeholderDigest + `"}]}` + + _, err = deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(bare), rewritten.Declaration, func(string) {}) + if err == nil { + t.Fatal("a control plane that can open nothing was accepted") + } + if !strings.Contains(err.Error(), storeVariablePrefix+""+storeFileSuffix) { + t.Errorf("the refusal does not say what shape is expected: %v", err) + } +} + +// The permanent control plane is asked a question, not merely looked at — the same question the +// temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so +// a reply proves the sealed connections it was given are the ones the substrate made. +func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "module list": "", + "exec mesh-control /mesh-control": "1 node, 0 waiting\n", + })} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + out, err := InstallControlPlane(context.Background(), + installing(t, catalogueWith(t, ControlPlaneModule, theControlPlaneModule)), + Deps{Run: runtime.run}, control, rewritten.Declaration, pushedReference, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Answered != "1 node, 0 waiting" { + t.Errorf("the permanent control plane's reply is reported as %q", out.Answered) + } + if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands) + } + // And the module was registered with the digest, not with the placeholder. + if !runtime.ran("module add /mesh-control-module.json") { + t.Errorf("the module was never registered: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go new file mode 100644 index 0000000..2387ec3 --- /dev/null +++ b/internal/bootstrap/enrol.go @@ -0,0 +1,276 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// hereIs what `node list` says about a machine the mesh has heard from recently. +// +// mesh-control prints one of three words per node: "here", "never spoken", or "out of touch ". +// The installer waits for the first, and it is the only honest proof that the host agent is +// running: an enrolled machine whose host is not running looks exactly like an enrolled machine +// whose host has crashed, and both look exactly like a successful install until the first push +// silently applies nothing. +const hereIs = "here" + +// Enrolled is what step 6 did. +type Enrolled struct { + // Node is the name this machine is known by. + Node string + // Added is true when the mesh had no record and one was created. + Added bool + // Joined is true when this machine enrolled during this run. False on a re-run. + Joined bool + // Agent says how the host came to be running: what was found, or what was started. + Agent string +} + +// Enrol makes this machine the mesh's first node, and gets the host agent running on it. +// +// **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a +// control plane that answers — a mesh of one node in the sense that it has one machine and zero +// node records. Everything after this point is the control plane being *told* things, and none of +// it reaches a machine until a host is running there to hear it. +// +// Four things, in this order, each asked before it is done: +// +// 1. a node record, unless `node list` already shows one +// 2. a one-time token, unless this machine already holds an identity +// 3. `mesh-host enrol`, which is the machine presenting the identity it already had +// 4. the host agent running, and the mesh saying it has heard from it +// +// **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an +// identity the mesh has recorded, and enrolling again would replace it with a second one the mesh +// does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer +// earlier and with a sentence about what to do. +// +// `control.run` is this machine's runner and not only the control plane's: the same injected +// Runner reaches the container, the service manager and the host binary, which is what lets the +// whole of this be tested without any of the three. +func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (Enrolled, error) { + + out := Enrolled{Node: o.Node} + if strings.TrimSpace(o.Node) == "" { + return out, errors.New( + "this machine has no name to be known by. Give one with --node; it is what the mesh " + + "records, what a token is issued against, and what every later assignment names") + } + + // 1. The record. + nodes, err := control.tell(ctx, "node", "list") + if err != nil { + return out, err + } + if mentions(nodes, o.Node) { + say(" already a node " + o.Node) + } else { + if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { + return out, err + } + out.Added = true + say(" node record " + o.Node) + } + + // 2 and 3. The identity, and being known. + // + // Asked of this machine's own identity file rather than of the mesh, because the two answer + // different questions: the mesh knows whether a record exists, and only the machine knows + // whether it holds the key that record names. + where := identity.Path(o.State) + switch mine, err := identity.Load(where); { + case err == nil && mine.Node == o.Node: + say(" already enrolled " + o.Node + " — " + where) + case err == nil: + return out, fmt.Errorf( + "this machine is already node %q and was asked to become %q.\n"+ + "Re-enrolling replaces the identity the mesh has recorded, so it is not something "+ + "an installer does on its own. Run with --node %s, or remove %s and start over "+ + "deliberately", mine.Node, o.Node, mine.Node, where) + case !errors.Is(err, identity.ErrNoIdentity): + return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err) + default: + said, err := control.tell(ctx, "token", "issue", "--node", o.Node) + if err != nil { + return out, err + } + token, err := tokenIn(said) + if err != nil { + return out, err + } + joining, cancel := context.WithTimeout(ctx, o.Wait) + joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) + cancel() + if err != nil { + return out, fmt.Errorf( + "%s would not enrol this machine: %w\n%s\n"+ + "The token is one-time and has now been spent; running this again issues "+ + "another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined))) + } + if !strings.Contains(joined, "enrolled as "+o.Node) { + // Exit zero and no such sentence. Refused rather than believed: the host says exactly + // this line on success, and something that succeeded without saying it did something + // else. + return out, fmt.Errorf( + "%s exited happily and did not say it enrolled as %s:\n%s", + o.Host, o.Node, indent(strings.TrimSpace(joined))) + } + out.Joined = true + say(" enrolled as " + o.Node) + } + + // 4. The agent. + agent, err := runTheHost(ctx, o, sys, control, say) + if err != nil { + return out, err + } + out.Agent = agent + return out, nil +} + +// runTheHost makes sure something on this machine is listening to the mesh, and proves it. +// +// **The installer does not install the service, and says so.** A unit file is a packaging decision +// — where the binary lives, which user it runs as, what it is called — and an installer that +// invented one would be putting a file on the machine that whatever installed `mesh-host` will +// later disagree with. So this starts a unit that is already there and refuses when there is none. +// +// It goes through the host's OWN system abstraction rather than running systemctl itself, for the +// reason `ApplyBundle` gives about applying: two implementations of "is this service running" is +// how the installer and the host come to disagree about a machine. It also gets the right refusal +// for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped", +// which is exactly the distinction that matters here. +// +// --host-in-background is the lab's arrangement, kept because the lab is what exercises this and +// it has no service. It is loud about what it is, because a host started this way is gone at the +// next reboot and the mesh would go quiet for reasons nobody would connect to an install. +func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (string, error) { + + // Asked first, and of the mesh rather than of the process table. What matters is not that a + // process exists but that the mesh has heard from it, and only one of those two is the thing + // every later step depends on. + if heard, err := heardFrom(ctx, control, o.Node); err != nil { + return "", err + } else if heard { + say(" host running the mesh has heard from " + o.Node) + return "already running", nil + } + + how := "" + switch { + case o.HostInBackground: + // Detached, and not waited for: this process runs until the machine stops, so a runner + // that captures output would never return. `nohup … &` through a shell is how the lab + // starts it and is deliberately the same command. + started, cancel := context.WithTimeout(ctx, o.Timeout) + _, err := control.run(started, "sh", "-c", + fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State)) + cancel() + if err != nil { + return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err) + } + how = "started in the background" + say(" host running started in the background — THIS DOES NOT SURVIVE A REBOOT.") + say(" A real machine needs " + o.HostService + " installed and enabled.") + + default: + state, err := sys.ServiceState(ctx, control.run, o.HostService) + if err != nil { + return "", fmt.Errorf( + "the mesh has not heard from %s and this machine has no %s to start: %w\n"+ + "The host has to be running for anything the mesh says to reach this machine. "+ + "Install the service that supervises it and run this again — every step "+ + "before this one will say it is already done. In a lab, --host-in-background "+ + "starts it unsupervised instead, and that is not an install", + o.Node, o.HostService, err) + } + if state != "running" { + if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil { + return "", fmt.Errorf("cannot start %s: %w", o.HostService, err) + } + how = "started " + o.HostService + say(" host running started " + o.HostService) + } else { + // Running, and the mesh has not heard from it. Not an error yet — it may have started + // a second ago — so it is waited for below like everything else that is merely + // starting. + how = o.HostService + " was already running" + say(" host running " + o.HostService + " is running") + } + // At boot as well, or the machine comes back without a mesh and nothing says why. + if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" { + if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil { + return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err) + } + say(" host at boot " + o.HostService + " enabled") + } + } + + // Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node + // are two different facts, and only the second is the one every later step rests on. + deadline := time.Now().Add(o.Wait) + for { + heard, err := heardFrom(ctx, control, o.Node) + if err != nil { + return "", err + } + if heard { + say(" the mesh hears " + o.Node) + return how, nil + } + if time.Now().After(deadline) { + return "", fmt.Errorf( + "%s is running and the mesh has not heard from %s after %s.\n"+ + "The host links to the broker at the address the token carried — if that "+ + "address is not one this machine can reach, this is where it shows. Read the "+ + "host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+ + "installer wrote", o.HostService, o.Node, o.Wait) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } +} + +// heardFrom asks the mesh whether this node has spoken to it. +func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) { + listing, err := control.tell(ctx, "node", "list") + if err != nil { + return false, err + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == node { + return fields[1] == hereIs, nil + } + } + return false, nil +} + +// tokenIn finds the token in what `token issue` said. +// +// The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation +// around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies +// it without this having to know the format — which is what keeps the installer from having an +// opinion about a thing the control plane owns. +func tokenIn(said string) (string, error) { + for _, line := range strings.Split(said, "\n") { + line = strings.TrimSpace(line) + if len(line) > 100 && !strings.ContainsAny(line, " \t") { + return line, nil + } + } + return "", fmt.Errorf( + "the mesh issued a token and there is no token in what it said:\n%s", + indent(strings.TrimSpace(said))) +} diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go new file mode 100644 index 0000000..7561d56 --- /dev/null +++ b/internal/bootstrap/enrol_test.go @@ -0,0 +1,223 @@ +package bootstrap + +import ( + "context" + "crypto/ed25519" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// Step 6 is where the mesh stops being something running on a machine and starts being something +// the machine belongs to. What these tests defend is that it cannot happen twice, and that +// "installed" is never claimed for a machine the mesh has not actually heard from. + +// alreadyEnrolled writes an identity file, as `mesh-host enrol` leaves behind. +func alreadyEnrolled(t *testing.T, node string) string { + t.Helper() + state := filepath.Join(t.TempDir(), "state.json") + mine, err := identity.Generate(node) + if err != nil { + t.Fatal(err) + } + // A whole membership, because an identity that cannot reach its mesh is refused on the way in + // — which is the right refusal and not the one being tested here. + signer, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{ + Broker: "192.0.2.10:5671", Fingerprint: "sha256:whatever", + Signer: signer, Password: "issued-at-enrolment", + } + if err := identity.Save(identity.Path(state), mine); err != nil { + t.Fatal(err) + } + return state +} + +func arch(t *testing.T) system.System { + t.Helper() + chosen, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + return chosen +} + +// A machine that has already enrolled is not enrolled again, and no token is spent on it. The +// installer is run over and over; a second identity is one the mesh does not know, and the mesh +// believes the first. +func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor here 01J0\n", nil + } + return "", fmt.Errorf("unexpected: %s %v", name, args) + }} + + out, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Joined || out.Added { + t.Error("a machine that had already enrolled enrolled again") + } + if runtime.ran("token issue") { + t.Errorf("a token was issued for a machine that already holds an identity: %v", + runtime.commands) + } + if out.Agent != "already running" { + t.Errorf("the host agent is reported as %q", out.Agent) + } +} + +// A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling +// replaces the identity the mesh recorded, which is a deliberate act and not something an +// installer does on its own. +func TestAMachineEnrolledUnderAnotherNameIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "node list") { + return "somewhere-else here 01J0\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "somewhere-else"), Timeout: time.Second, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine already enrolled as something else was enrolled again") + } + for _, wanted := range []string{"somewhere-else", "--node"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// **The mesh having heard from the node is the proof, not a process existing.** A host that is +// running and cannot reach the broker looks exactly like a successful install until the first push +// silently applies nothing — which is the class of fault this whole program exists to stop being +// found late. +func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + // Enrolled, and never spoken. + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + return "LoadState=loaded\nActiveState=active\n", nil + case args[0] == "is-enabled": + return "enabled\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a node the mesh has never heard from was reported enrolled and running") + } + if !strings.Contains(err.Error(), "MESH_BROKER_ADDRESS") { + t.Errorf("the failure does not name the thing that is silently fatal when wrong:\n%v", err) + } +} + +// A machine with no service to start is refused, and the refusal says what is missing rather than +// inventing a unit file. What a unit says is a packaging decision, and an installer writing one +// would put a file on the machine that whatever installed the host will disagree with. +func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + // systemd knows nothing about it. + return "LoadState=not-found\nActiveState=inactive\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine with no host service was reported as having a running host") + } + for _, wanted := range []string{"mesh-host.service", "--host-in-background"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// A machine with no name is refused before anything is said to the mesh. The name is what the +// record, the token, the assignment and the push all name, and one the installer invented would +// match nothing anybody types anywhere else. +func TestAMachineWithNoNameIsRefusedBeforeAnythingIsAsked(t *testing.T) { + runtime := &asked{answer: func(string, []string) (string, error) { + return "", fmt.Errorf("nothing should have been asked") + }} + _, err := Enrol(context.Background(), Options{Timeout: time.Second}, arch(t), + controlPlane{container: "temp-mesh-control", run: runtime.run}, func(string) {}) + if err == nil { + t.Fatal("a machine with no name was enrolled") + } + if len(runtime.commands) != 0 { + t.Errorf("the mesh was asked something first: %v", runtime.commands) + } +} + +// The token is found in what the mesh said, by the rule the lab uses: the one long unbroken line. +// The installer does not parse a format the control plane owns. +func TestTheTokenIsFoundInWhatTheMeshSaid(t *testing.T) { + said := "a token for anchor, good once:\n\n " + strings.Repeat("t", 240) + "\n\n" + + "carry it to the machine and run: mesh-host enrol --token \n" + token, err := tokenIn(said) + if err != nil { + t.Fatal(err) + } + if token != strings.Repeat("t", 240) { + t.Errorf("the token was read as %q", token) + } + + if _, err := tokenIn("nothing here that looks like one\n"); err == nil { + t.Fatal("an answer with no token in it was accepted") + } +} + +// A listing's name is matched as a whole word at the start of a line, so `registry` is not found +// inside `registry-mirror`. A substring match would report a module installed that is not, and the +// installer would skip creating it. +func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) { + listing := "registry-mirror 1 built abc\nother 1 built def\n" + if mentions(listing, "registry") { + t.Error("registry-mirror was read as registry") + } + if !mentions(listing, "registry-mirror") { + t.Error("registry-mirror was not found") + } +} diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go new file mode 100644 index 0000000..2c6eb43 --- /dev/null +++ b/internal/bootstrap/module.go @@ -0,0 +1,145 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +// placeholderDigest is what the catalogue writes where a built image's digest will go. +// +// Sixty-four zeros. A manifest in the catalogue names an image the mesh builds and pushes, and +// until that has happened there is no digest to name — so the convention is a digest that is +// obviously not one, replaced by the pipeline when it publishes. The installer meets it twice: it +// must NOT be there in the registry's manifest, whose image is upstream and never built +// (novox/hq 04-ISSUES/029), and it MUST be there in the control plane's, which is the image +// step 8 has just pushed. +const placeholderDigest = "sha256:" + "0000000000000000000000000000000000000000000000000000000000000000" + +// catalogueDir is where a mesh-catalog checkout keeps its manifests. +const catalogueDir = "modules" + +// manifestFile is the path a module's manifest is read from, given a catalogue checkout. +func manifestFile(catalogue, module string) string { + return filepath.Join(catalogue, catalogueDir, module, "module.json") +} + +// readManifest reads one module's manifest out of a mesh-catalog checkout. +// +// **From a checkout rather than from anything the mesh serves**, and that is the ordering the whole +// pivot exists to respect: at this point the mesh has no build machine, no forge and — until step 7 +// finishes — no registry. What a manifest is, is a file; the installer is handed the directory it +// is in and reads it. +func readManifest(catalogue, module string) ([]byte, error) { + path := manifestFile(catalogue, module) + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf( + "the %s module's manifest could not be read: %w\n"+ + "--catalog is a checkout of the mesh's catalogue repository, and this is read from "+ + "%s inside it", module, err, filepath.Join(catalogueDir, module, "module.json")) + } + return raw, nil +} + +// Installed is what installing one module did. +type Installed struct { + // Module is its name. + Module string + // Known is true when the mesh already had this module in its catalogue. The manifest is + // registered either way — a re-run with a changed manifest must land — so this reports what + // was found rather than what was skipped. + Known bool + // Assigned is true when this node was given the module during this run. + Assigned bool + // Pushed is what the mesh said when it sent this node its declaration. + Pushed string +} + +// installModule registers a manifest, gives it to this node, and sends it. +// +// The three verbs a person types, in the order they type them, through the same commands. There is +// no installer-only path into the mesh: everything here is `module add`, `assign` and `push`, so +// what the installer does on a bare machine and what an operator does on a running mesh are the +// same act (novox/hq ADR 0035, one refusal per act however it is asked for). +func installModule(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out, err := registerAndAssign(ctx, o, control, module, manifest, say) + if err != nil { + return out, err + } + out.Pushed, err = pushNode(ctx, o, control, say) + return out, err +} + +// registerAndAssign is the half of installing that happens before anything is sent. +// +// Split out because one module needs something in between: the control plane's own store +// connections have to be accepted before its declaration is composed, or the mesh would seal +// thirty-two random bytes into the file it expects a connection string in and the container would +// come up unable to open anything (mesh-control's `secret accept`, and what it exists for). +// +// **`module add` is run every time and is not skipped when the module is already known.** It is an +// upsert on the manifest, and the manifest is exactly what changes between runs — step 9 registers +// the control plane with a digest that did not exist the first time. Skipping it because the name +// was already in the catalogue would silently pin the mesh to the previous image. +func registerAndAssign(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out := Installed{Module: module} + + known, err := control.tell(ctx, "module", "list") + if err != nil { + return out, err + } + out.Known = mentions(known, module) + + remote := "/" + module + "-module.json" + if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { + return out, err + } + if _, err := control.tell(ctx, "module", "add", remote); err != nil { + return out, err + } + if out.Known { + say(" registered " + module + " — the mesh already knew it; the manifest is now this one") + } else { + say(" registered " + module) + } + + assigned, err := control.tell(ctx, "assign", o.Node, module) + if err != nil { + return out, err + } + out.Assigned = true + say(" assigned " + module + " to " + o.Node) + if refusal := strings.TrimSpace(assigned); strings.Contains(refusal, "but ") { + // `assign` records what a person meant and says at once when the machine cannot host it. + // Repeated rather than swallowed: the push below will apply everything else and this is + // the only place the reason appears. + say(indent(refusal)) + } + + return out, nil +} + +// pushNode sends this node everything it should be. +// +// Given the long wait rather than the probe timeout: a push composes every declaration this node +// should hold, seals every secret in them and publishes them, and on a first node that is the +// slowest thing the mesh does. +func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + said, err := control.within(o.Wait).tell(ctx, "push", o.Node) + if err != nil { + return "", fmt.Errorf( + "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ + "not been sent it. Nothing is half-applied — a push the mesh refused sent nothing "+ + "at all. Fix what it named and run this installer again: it will find the module "+ + "already registered and try the push again", err) + } + say(" pushed " + o.Node) + return strings.TrimSpace(said), nil +} diff --git a/internal/bootstrap/publish.go b/internal/bootstrap/publish.go new file mode 100644 index 0000000..e7288b1 --- /dev/null +++ b/internal/bootstrap/publish.go @@ -0,0 +1,173 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// ControlPlaneRepository is what the control plane's image is called in the mesh's own registry. +const ControlPlaneRepository = "mesh-control" + +// genesisTag is the tag the first push uses. +// +// A tag is not a pin and is never what anything is deployed from — the digest the registry assigns +// is (novox/hq ADR 0006). This exists so a person reading `/v2/mesh-control/tags/list` can see +// which image this mesh started from, and so the push has something to name. Everything downstream +// uses the digest that comes back. +const genesisTag = "genesis" + +// Published is what step 8 did. +type Published struct { + // Reference is `/mesh-control@sha256:…` — the first manifest digest this image has + // ever had, and the thing that makes the control plane an ordinary module. + Reference string + // Tagged is where it was pushed, tag and all. + Tagged string + // Already is true when the registry was already serving it and nothing was pushed. + Already bool +} + +// PublishControlPlane puts the carried image into the mesh's own registry and reads back its digest. +// +// **This is the pivot's hinge.** Every image must be pinned by digest, and a digest a pin can mean +// is one a REGISTRY assigned when something was pushed to it. The control plane's image is built +// from source and pushed nowhere, so it has none — which is why the substrate names it by the +// digest of its own configuration, and why that is legal exactly where nothing could have served +// one. The moment this push completes, that stops being true: the image has a manifest digest, so +// the control plane can be named the way every other module is named, so the mesh can build and +// roll out its own upgrades. If this step is skipped the machine still works and the mesh cannot +// upgrade itself, which is the check novox/hq ADR 0067 states: after installing, the running +// control plane must be pinned by a digest the mesh's own registry assigned, not by an image id. +// +// **It mirrors mesh-control's `internal/builder`.PublishImage rather than importing it.** Tag, +// push, read back `RepoDigests`, refuse anything without `@sha256:` — the same four steps, because +// there is exactly one right way to learn what a registry will serve something as, and it is to +// ask the registry. It is not imported because that code is tier 2: the host and its installer +// depend on nothing that must be installed first (novox/hq ADR 0041), and taking a dependency on +// the control plane's repository to raise the control plane would be the cycle this whole ADR is +// about, one layer up. The duplication is four commands, and it is deliberate. +// +// One difference, and it is a correction rather than a divergence: the digest is chosen from +// `RepoDigests` by repository instead of taken as element zero. An image that has been pushed to +// more than one registry has more than one entry, and element zero is then whichever the runtime +// happened to list first — which would pin this mesh to somebody else's registry, silently. +func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string, + say func(string)) (Published, error) { + + remote := o.Registry + "/" + ControlPlaneRepository + out := Published{Tagged: remote + ":" + genesisTag} + + // Asked first. A digest already served is a fact about the registry, and re-pushing an image + // the registry already holds is asking it to store what it already has under the name it + // already has. + if held, err := digestOf(ctx, o, d, remote); err != nil { + return out, err + } else if held != "" { + out.Reference, out.Already = held, true + say(" already published " + held) + return out, nil + } + + if _, err := d.Run(ctx, "docker", "tag", imageID, out.Tagged); err != nil { + return out, fmt.Errorf("cannot tag the carried image as %s: %w", out.Tagged, err) + } + if _, err := d.Run(ctx, "docker", "push", out.Tagged); err != nil { + return out, fmt.Errorf( + "the container runtime would not push %s: %w\n"+ + "The registry is plain HTTP and wants no credentials, deliberately — it is reached "+ + "over the mesh's own network, which is already the encrypted and authenticated "+ + "thing. A runtime refusing it for being insecure is refusing a registry on %s, "+ + "which it does not do for a loopback address", + out.Tagged, err, o.Registry) + } + + // Read back, from the registry's own answer rather than computed here. What matters is what + // the registry will serve for that reference, and only it can say (novox/hq ADR 0018). + pinned, err := digestOf(ctx, o, d, remote) + if err != nil { + return out, err + } + if pinned == "" { + return out, fmt.Errorf( + "%s was pushed and the registry does not serve it.\n"+ + "The next step names the control plane's module by the digest this was supposed to "+ + "produce, so there is nothing to name. Check `docker push` and "+ + "http://%s/v2/%s/tags/list", out.Tagged, o.Registry, ControlPlaneRepository) + } + out.Reference = pinned + say(" published " + pinned) + return out, nil +} + +// digestOf is what the registry serves this repository as, or empty if it serves it at all. +// +// Both halves are asked, because either alone lies. The registry's tag list says something was +// pushed and not what its digest is; the runtime's `RepoDigests` says what a digest was and not +// whether the registry still has it — a registry whose volume was recreated would leave the +// runtime remembering a digest nothing serves, and the module registered against it would pin the +// mesh to an image that cannot be pulled. +func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, body, err := d.Fetch(asking, + "http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list") + cancel() + if err != nil { + return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err) + } + if status == http.StatusNotFound { + // Nothing has ever been pushed under this name. An answer, not a failure. + return "", nil + } + if status != http.StatusOK { + return "", fmt.Errorf("the registry answered %d when asked what it holds for %s", + status, ControlPlaneRepository) + } + var listed struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &listed); err != nil { + return "", fmt.Errorf("the registry's answer about %s is not readable: %w", + ControlPlaneRepository, err) + } + if !contains(listed.Tags, genesisTag) { + return "", nil + } + + // The registry has it. What digest, according to the runtime that pushed it. + reading, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := d.Run(reading, "docker", "inspect", "--format", "{{json .RepoDigests}}", + remote+":"+genesisTag) + cancel() + if err != nil { + // The registry holds the tag and this machine's runtime does not hold the image. That + // happens on a re-run after the image was pruned, and it is not something to work around + // by trusting the tag: a tag can be made to point elsewhere. + return "", nil + } + var digests []string + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &digests); err != nil { + return "", fmt.Errorf("the runtime's answer about %s is not readable: %w", remote, err) + } + for _, digest := range digests { + if !strings.HasPrefix(digest, remote+"@sha256:") { + // Somebody else's registry serving the same image. Skipped rather than used: pinning + // this mesh's control plane to a registry it does not run is exactly the dependency + // the pivot exists to remove. + continue + } + return digest, nil + } + return "", nil +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} diff --git a/internal/bootstrap/publish_test.go b/internal/bootstrap/publish_test.go new file mode 100644 index 0000000..759414d --- /dev/null +++ b/internal/bootstrap/publish_test.go @@ -0,0 +1,197 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" +) + +// Step 8 is the pivot's hinge (novox/hq ADR 0067): the carried image gets a manifest digest, which +// is the first one it has ever had, and that is what lets the control plane be named the way every +// other module is named. These tests defend how that digest is learned, because a wrong one pins +// the mesh to an image nothing on this machine serves. + +func publishing(t *testing.T, fetch func(string) (int, string, error), + run func(name string, args []string) (string, error)) (Options, Deps, *asked) { + t.Helper() + runtime := &asked{answer: run} + return Options{ + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + }, Deps{ + Run: runtime.run, + Fetch: func(_ context.Context, url string) (int, string, error) { + return fetch(url) + }, + }, runtime +} + +// Nothing has ever been pushed under this name, so the registry says 404 — and that is an answer, +// not a failure. An installer that treated it as one would refuse on the first run of the step it +// exists to perform. +func TestAnImageNoRegistryHasEverHeldIsPushed(t *testing.T) { + pushed := false + o, d, runtime := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + switch args[0] { + case "tag": + return "", nil + case "push": + pushed = true + return "", nil + case "inspect": + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("a", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Already { + t.Error("an image no registry held was reported as already published") + } + if !strings.HasPrefix(out.Reference, "127.0.0.1:5000/mesh-control@sha256:") { + t.Errorf("the control plane is pinned as %q", out.Reference) + } + if !runtime.ran("docker push 127.0.0.1:5000/mesh-control:genesis") { + t.Errorf("nothing was pushed: %v", runtime.commands) + } +} + +// An image the registry already serves is not pushed again, and says so. Blobs are named by their +// content, so re-pushing is asking a registry to store what it already has under the name it +// already has — and the installer is run over and over. +func TestAnImageTheRegistryAlreadyServesIsNotPushedAgain(t *testing.T) { + o, d, runtime := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("b", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("an image the registry already serves was not reported as already published") + } + if runtime.ran("docker push") { + t.Errorf("it was pushed again: %v", runtime.commands) + } +} + +// **The digest is chosen by repository, not taken as element zero.** An image that has been pushed +// to more than one registry has more than one entry, and element zero is whichever the runtime +// listed first — which would pin this mesh's control plane to somebody else's registry, silently, +// which is the dependency the whole pivot exists to remove. +func TestTheDigestComesFromThisMeshsOwnRegistry(t *testing.T) { + elsewhere := "some.other.registry/mesh-control@sha256:" + strings.Repeat("c", 64) + ours := "127.0.0.1:5000/mesh-control@sha256:" + strings.Repeat("d", 64) + + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["` + elsewhere + `","` + ours + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Reference != ours { + t.Errorf("the control plane is pinned as %q, and this mesh's registry serves %q", + out.Reference, ours) + } +} + +// A push that produced no digest this mesh's registry serves is refused, and the refusal says what +// depends on it. The next step names the control plane's module by that digest, so there would be +// nothing to name — and finding that out one step later would mean registering a module pinned to +// an empty string. +func TestAPushThatProducedNoDigestIsRefused(t *testing.T) { + pushed := false + o, d, _ := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + // Pushed, and the registry still does not list it. + return http.StatusOK, `{"tags":[]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "push" { + pushed = true + } + return "", nil + }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a push that produced no digest was accepted") + } + if !strings.Contains(err.Error(), "does not serve it") { + t.Errorf("the refusal does not say what is missing: %v", err) + } +} + +// A tag is not a pin. If the runtime answers with something that is not pinned by digest, it is +// not used — a tag can be made to point at a different image, and this reference is applied on +// machines with no mesh to ask about anything (novox/hq ADR 0006). +func TestATagIsNotAPin(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control:genesis"]`, nil + } + return "", nil + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatalf("a tag was accepted as a pin: %q", out.Reference) + } +} + +// A registry that cannot be reached at all is said so plainly rather than becoming a push that +// fails for a reason nobody can read. +func TestARegistryThatCannotBeAskedIsSaidSo(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return 0, "", errors.New("connection refused") + }, + func(string, []string) (string, error) { return "", nil }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a registry that refused the connection was treated as empty") + } + if !strings.Contains(err.Error(), "cannot ask the registry") { + t.Errorf("the refusal does not say the registry could not be asked: %v", err) + } +} diff --git a/internal/bootstrap/registry.go b/internal/bootstrap/registry.go new file mode 100644 index 0000000..637d9e7 --- /dev/null +++ b/internal/bootstrap/registry.go @@ -0,0 +1,204 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// RegistryModule is the module that provides the mesh's artifact store. +const RegistryModule = "registry" + +// registryResource is the id of the resource in that module's manifest that runs the registry. +// What the container is CALLED is read from the manifest rather than assumed, because the name is +// the catalogue's to choose and the installer only has to know which resource to wait for. +const registryResource = "store" + +// Registry is what step 7 did. +type Registry struct { + Installed + // Container is the container the manifest declares, confirmed running. + Container string + // Address is host:port the registry answers on, as this machine reaches it. + Address string + // Answered is the status the registry's own `/v2/` gave back. + Answered int +} + +// InstallRegistry gives this mesh somewhere to put images. +// +// **Its image is upstream and it is never built.** novox/hq 04-ISSUES/029 is the whole reason this +// step exists in this position: a module that provides the artifact store cannot be delivered +// through the artifact store, so the registry is the one module whose image is pulled from the +// internet like the store and the broker before it. A manifest carrying the catalogue's +// placeholder digest here would mean somebody had made it buildable, which is the cycle again — +// so it is refused rather than pulled. +// +// **No credentials, and that is deliberate.** The registry is reached over the mesh's own private +// network, which is already the encrypted and authenticated thing; a second layer inside it would +// be certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder`, which pushes to it the same way). So there is nothing here to configure and +// nothing to seal — which is also why step 8 can push without the mesh having issued anything. +// +// **It is verified by asking it, not by looking at it.** A container that is up is not a registry +// that serves: `/v2/` is the registry API's own "yes, I am one and I am ready", and it is the +// question step 8 depends on the answer to. +func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlane, + say func(string)) (Registry, error) { + + out := Registry{Address: o.Registry} + + manifest, err := readManifest(o.Catalogue, RegistryModule) + if err != nil { + return out, err + } + container, image, err := containerIn(manifest, registryResource) + if err != nil { + return out, err + } + if strings.Contains(image, placeholderDigest) { + return out, fmt.Errorf( + "the %s module's image is %q, which is the catalogue's placeholder for something the "+ + "mesh builds and pushes.\n"+ + "This module is the one that cannot work that way: it PROVIDES the place built "+ + "images go, so it can never be delivered through it (novox/hq 04-ISSUES/029). Its "+ + "image is upstream and pinned in the manifest", RegistryModule, image) + } + out.Container = container + say(" registry image " + image + " — upstream, never built") + + installed, err := installModule(ctx, o, control, RegistryModule, manifest, say) + out.Installed = installed + if err != nil { + return out, err + } + + // Read back, in two stages, because they fail differently. A container that never appears is + // a declaration that did not reach this node or an image that would not pull; a container + // that is up and does not answer is a registry that started and failed. + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + status, err := waitForTheRegistry(ctx, d, o, say) + if err != nil { + return out, err + } + out.Answered = status + return out, nil +} + +// waitForTheRegistry asks `/v2/` until it answers. +func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) { + where := "http://" + o.Registry + "/v2/" + + deadline := time.Now().Add(o.Wait) + var last string + for { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, _, err := d.Fetch(asking, where) + cancel() + switch { + case err != nil: + last = err.Error() + case status == http.StatusOK: + say(fmt.Sprintf(" replies %s answered %d", where, status)) + return status, nil + default: + // A registry that answers 401 is one that wants credentials, which this one is + // configured not to. Reported as what it said rather than retried into a timeout. + last = fmt.Sprintf("it answered %d", status) + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(answerEvery): + } + } + return 0, fmt.Errorf( + "the registry's container is running and %s does not answer, after waiting %s: %s\n"+ + "Running is not serving. `/v2/` is the registry API saying it is ready, and the next "+ + "step pushes the control plane's image to it — so this is refused here rather than "+ + "discovered inside a `docker push`. `docker logs mesh-registry` says what it did", + where, o.Wait, last) +} + +// waitForContainer waits for a container the mesh was asked to create to be running. +// +// Unlike the substrate's own verify, this one waits: the mesh applies through a node's host, over +// the broker, asynchronously. A push that the control plane accepted has not yet happened on the +// machine, and refusing on the first look would refuse every correct install. +func waitForContainer(ctx context.Context, run Runner, probe, wait time.Duration, name string, + say func(string)) error { + + deadline := time.Now().Add(wait) + var last string + for { + state, err := containerRunning(ctx, run, probe, name) + switch { + case err != nil: + last = "it is not there at all" + case state.running: + say(" running " + name) + return nil + default: + last = "it is " + state.status + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + return fmt.Errorf( + "the mesh accepted the push and %q is not running after %s: %s\n"+ + "The control plane sends a declaration over the broker and this node's host applies "+ + "it, so the two ends fail differently: `mesh-host` on this machine says what it made "+ + "of the declaration, and `status` on the control plane says whether it was collected "+ + "at all", name, wait, last) +} + +// containerIn finds one container resource in a module manifest and gives back its name and image. +// +// It reads the manifest as data rather than through the catalogue's own parser, because that +// parser lives in the control plane and the host depends on nothing installed first +// (novox/hq ADR 0041) — importing it would put tier 2 inside tier 0. What is read here is two +// fields of a shape the catalogue owns; the manifest is handed to the control plane unchanged, and +// it is the control plane's `module add` that judges whether it is a manifest at all. +func containerIn(manifest []byte, id string) (name, image string, err error) { + var m struct { + Module string `json:"module"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Image string `json:"image"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "", "", fmt.Errorf("this manifest is not readable as JSON: %w", err) + } + var containers []string + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + containers = append(containers, r.ID) + if r.ID == id { + return r.Name, r.Image, nil + } + } + return "", "", fmt.Errorf( + "the %s module declares no container %q, so the installer does not know what to wait for. "+ + "It declares: %s", m.Module, id, strings.Join(containers, ", ")) +} diff --git a/internal/bootstrap/registry_test.go b/internal/bootstrap/registry_test.go new file mode 100644 index 0000000..52f923d --- /dev/null +++ b/internal/bootstrap/registry_test.go @@ -0,0 +1,216 @@ +package bootstrap + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Step 7 installs the one module whose image can never come from the mesh's own registry, because +// it IS the mesh's own registry (novox/hq 04-ISSUES/029). These tests defend that, and defend the +// distinction the whole verify layer of this program is built on: a container that is up is not a +// service that answers. + +// catalogueWith writes a fake catalogue checkout holding one module's manifest. +// +// A fixture here rather than the real catalogue, unlike the substrate example the rewrite tests +// use: the catalogue is a different repository on a different branch, and a test that read it +// would pass or fail according to what somebody else had checked out. +func catalogueWith(t *testing.T, module, manifest string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, catalogueDir, module) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +const upstreamRegistryManifest = `{ + "module": "registry", + "version": "1", + "provides": [{"name": "artifact-store", "scope": "mesh"}], + "capabilities": ["container-runtime"], + "resources": [ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}, + {"id": "store", "type": "container", "name": "mesh-registry", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "ports": ["5000:5000"]} + ] +}` + +// aMeshThatAgrees answers every command the installer issues at steps 7 and 9 the way a working +// mesh would, except for whatever a test overrides. +func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, error) { + return func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + for fragment, said := range answers { + if strings.Contains(joined, fragment) { + return said, nil + } + } + switch { + case name != "docker": + return "", fmt.Errorf("unexpected program %q", name) + case args[0] == "cp": + return "", nil + case args[0] == "inspect": + return "true running\n", nil + case args[0] == "exec": + return "", nil + } + return "", fmt.Errorf("unexpected: %v", args) + } +} + +func installing(t *testing.T, catalogue string) Options { + t.Helper() + return Options{ + Node: "anchor", + Catalogue: catalogue, + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + } +} + +// A container that is up is not a registry that serves. `/v2/` is the registry API's own "yes, I +// am one and I am ready", and the step after this pushes to it — so it is refused here rather than +// discovered inside a `docker push`. +func TestARegistryContainerThatIsUpIsNotARegistryThatServes(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusInternalServerError, "", nil + }, + } + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a registry whose container is up and which answers 500 was accepted") + } + for _, wanted := range []string{"/v2/", "Running is not serving"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// The whole of step 7, against a mesh that agrees: registered, assigned, pushed, up, and answering. +func TestARegistryThatAnswersIsAccepted(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusOK, "{}", nil + }, + } + + out, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Container != "mesh-registry" { + t.Errorf("the registry's container is %q", out.Container) + } + if out.Answered != http.StatusOK { + t.Errorf("the registry answered %d", out.Answered) + } + // Registered, assigned and pushed, through the same three commands a person types. + for _, wanted := range []string{ + "module add /registry-module.json", + "assign anchor registry", + "push anchor", + } { + if !runtime.ran(wanted) { + t.Errorf("the installer never ran %q: %v", wanted, runtime.commands) + } + } +} + +// **The registry's image is upstream and it is never built.** A manifest carrying the catalogue's +// placeholder digest would mean somebody had made this module buildable — which is the cycle +// novox/hq 04-ISSUES/029 settled: a module that provides the artifact store cannot be delivered +// through the artifact store. +func TestARegistryManifestThatWantsBuildingIsRefused(t *testing.T) { + wants := strings.Replace(upstreamRegistryManifest, + "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "mesh-runtime-registry@"+placeholderDigest, 1) + + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, wants)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a registry manifest naming an image the mesh would have to build was accepted") + } + if !strings.Contains(err.Error(), "04-ISSUES/029") { + t.Errorf("the refusal does not name the decision it rests on: %v", err) + } + if runtime.ran("module add") { + t.Error("it was registered anyway") + } +} + +// A catalogue that is not there is said plainly, with what --catalog is. This is the most likely +// mistake anybody makes at this step and the least interesting to debug. +func TestACatalogueThatIsNotThereIsSaidPlainly(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, filepath.Join(t.TempDir(), "nowhere")), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a catalogue that does not exist was accepted") + } + if !strings.Contains(err.Error(), "--catalog") { + t.Errorf("the refusal does not say what to fix: %v", err) + } +} + +// A refusal from the control plane is repeated verbatim. mesh-control refuses in paragraphs — +// "nothing provides route, wanted by registry" — and an installer that reported "exit status 1" +// would throw away the only thing a person can act on. +func TestWhatTheMeshRefusedIsRepeated(t *testing.T) { + refusal := "nothing provides \"route\", wanted by registry" + runtime := &asked{answer: func(name string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "push") { + return refusal, fmt.Errorf("exit status 1") + } + return aMeshThatAgrees(nil)(name, args) + }} + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run, + timeout: time.Second}, func(string) {}) + if err == nil { + t.Fatal("a push the mesh refused was reported as successful") + } + if !strings.Contains(err.Error(), refusal) { + t.Errorf("what the mesh said is not in the failure:\n%v", err) + } + if !strings.Contains(err.Error(), "run this installer again") { + t.Errorf("the failure does not say a re-run continues from here:\n%v", err) + } +} diff --git a/internal/bootstrap/retire.go b/internal/bootstrap/retire.go new file mode 100644 index 0000000..a84055d --- /dev/null +++ b/internal/bootstrap/retire.go @@ -0,0 +1,291 @@ +package bootstrap + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Retired is what step 10 did. +type Retired struct { + // Container is the temporary control plane that was dropped. + Container string + // Gone is true when the machine no longer has it. + Gone bool + // Already is true when it was gone before this step ran. + Already bool + // Bundle is where the bundle without it was written. + Bundle string + // Removed is how many resources the re-apply reported removing. + Removed int +} + +// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away. +// +// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The +// host owns what it has applied and removes what it owns and is no longer declared. So retiring the +// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the +// bundle is applied again, and the removal pass does what it does for every other resource that +// leaves a declaration. +// +// That is the whole of why the rename at step 3 mattered. Had the substrate and the module both +// called their container `mesh-control`, this apply would have removed the module's container — +// the host would have been asked to take away something it believed it owned, and it would have +// been right. Two names, two owners, and the removal is unambiguous. +// +// **It is the last step for a reason.** Until step 9 has a control plane that answers, the +// temporary one is the only thing that can tell this machine anything, and a machine left with no +// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one +// has been proved, and a run interrupted before it leaves two control planes, which is untidy and +// harmless — a re-run reaches this step and finishes. +func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System, + produced []byte, run Runner, say func(string)) (Retired, error) { + + out := Retired{Bundle: o.Out} + + current, err := declaration.ParseFileTrusted(produced) + if err != nil { + return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err) + } + temporary, err := controlPlaneIn(current) + if err != nil { + // The bundle already declares no control plane, which is what a re-run after this step + // finds. Nothing to drop, and nothing to be alarmed about. + say(" already dropped the bundle declares no temporary control plane") + out.Already = true + return out, nil + } + out.Container = temporary.Name + + // Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be + // READ, and a person coming to a machine after a pivot should be able to open the file the + // installer applied and see the substrate they recognise with the control plane gone from it. + // Re-serialising a parsed declaration would drop every comment in it. + bundle, err := removeResource(produced, ControlPlaneID) + if err != nil { + return out, err + } + without, err := declaration.ParseFileTrusted(bundle) + if err != nil { + return out, fmt.Errorf( + "taking the temporary control plane out of the bundle broke it: %w", err) + } + if _, err := controlPlaneIn(without); err == nil { + return out, fmt.Errorf( + "the bundle still declares %q after it was taken out, so nothing was removed and the "+ + "apply below would change nothing", ControlPlaneID) + } + if err := writeBundleFile(o.Out, bundle); err != nil { + return out, err + } + say(fmt.Sprintf(" wrote %s (%d resources) — without %s", + o.Out, len(without.Resources), temporary.Name)) + + // Applied the same way everything else here is applied, under the same origin, against the + // same state file. What makes this a removal rather than a no-op is that the state file + // records the container as something this installer applied, and the declaration no longer + // asks for it. + report, err := ApplyBundle(ctx, o, sys, without, run, say) + if err != nil { + return out, fmt.Errorf( + "%w\n\nThe permanent control plane is running and the temporary one is still here. "+ + "That is untidy and it is not broken: two control planes on one mesh are both "+ + "stateless and both correct. Run this installer again to finish", err) + } + for _, outcome := range report.Outcomes { + if outcome.Action == "removed" { + out.Removed++ + } + } + + // Read back. A removal that reported success and left the container running would leave two + // control planes consuming the same broker queues for ever, which is the state this step + // exists to end. + gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name) + if err != nil { + return out, err + } + out.Gone = gone + if !gone { + return out, fmt.Errorf( + "the apply reported the temporary control plane removed and %q is still running.\n"+ + "Two control planes are consuming this mesh's broker queues. Neither is wrong and "+ + "the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+ + "it, and this installer will then agree", temporary.Name, temporary.Name) + } + say(" gone " + temporary.Name) + return out, nil +} + +// removeResource takes one resource out of a bundle's text, comments and all. +// +// It walks the `resources` array counting braces, skipping over strings and comments so that a +// `//` inside a connection string is not read as the start of one — the substrate's own bundle +// contains `postgres://…` several times, and a scanner that did not know the difference would +// treat the rest of the line as a comment and lose a brace. +// +// What is removed is the element AND whatever precedes it back to the previous element, which is +// where the comment explaining it lives. A comment that outlives the thing it describes is worse +// than no comment: it is the file telling somebody the machine has a control plane it does not. +func removeResource(bundle []byte, id string) ([]byte, error) { + array := indexOutsideStrings(bundle, `"resources"`) + if array < 0 { + return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") + } + open := indexOutsideStrings(bundle[array:], "[") + if open < 0 { + return nil, fmt.Errorf("this bundle's resources are not a list") + } + open += array + + depth, from := 0, -1 + previous := open + inString, escaped, inLine, inBlock := false, false, false, false + for i := open + 1; i < len(bundle); i++ { + c := bundle[i] + switch { + case escaped: + escaped = false + case inString && c == '\\': + escaped = true + case inString: + if c == '"' { + inString = false + } + case inLine: + if c == '\n' { + inLine = false + } + case inBlock: + if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' { + inBlock, i = false, i+1 + } + case c == '"': + inString = true + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/': + inLine, i = true, i+1 + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*': + inBlock, i = true, i+1 + case c == '{': + if depth == 0 { + from = i + } + depth++ + case c == '}': + depth-- + if depth != 0 { + break + } + if isResource(bundle[from:i+1], id) { + return cut(bundle, previous, from, i+1), nil + } + previous = i + 1 + from = -1 + case c == ']' && depth == 0: + return nil, fmt.Errorf( + "this bundle declares no %q, so there is nothing to take out of it", id) + } + } + return nil, fmt.Errorf("this bundle's resources list does not end") +} + +// isResource reports whether one resource's text is the one wanted. +// +// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and +// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not. +func isResource(resource []byte, id string) bool { + var tight []byte + for _, c := range resource { + if c != ' ' && c != '\t' && c != '\n' && c != '\r' { + tight = append(tight, c) + } + } + return bytes.Contains(tight, []byte(`"id":"`+id+`"`)) +} + +// cut removes an element and what leads up to it, leaving the list valid. +// +// Whether the comma before or the comma after goes depends on where the element sits: an element +// with something before it takes the comma that joined them, and the first element takes the one +// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse — +// caught by the re-parse either way, and better not produced. +func cut(bundle []byte, previous, from, to int) []byte { + start := from + for i := previous; i < from; i++ { + if bundle[i] == ',' { + start = i + break + } + } + end := to + if start == from { + // Nothing before it, so the comma that follows is the one that would be left dangling. + for i := to; i < len(bundle); i++ { + if bundle[i] == ',' { + end = i + 1 + break + } + if bundle[i] == ']' { + break + } + } + } + out := make([]byte, 0, len(bundle)) + out = append(out, bundle[:start]...) + return append(out, bundle[end:]...) +} + +// indexOutsideStrings finds a fragment that is not inside a JSON string. +func indexOutsideStrings(haystack []byte, needle string) int { + inString, escaped := false, false + for i := 0; i < len(haystack); i++ { + switch { + case escaped: + escaped = false + continue + case haystack[i] == '\\' && inString: + escaped = true + continue + case haystack[i] == '"': + // The needle may itself start with a quote, so the match is tried before the quote is + // consumed. + if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + inString = !inString + continue + case inString: + continue + } + if bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + } + return -1 +} + +// isGone waits for a container to stop existing. +// +// Waited for rather than asked once, because a container being removed is a container that is +// stopping first, and a runtime answers about it until it has finished. +func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) { + deadline := time.Now().Add(wait) + for { + if _, err := containerRunning(ctx, run, probe, name); err != nil { + // The runtime does not know it. That is the answer being waited for. + return true, nil + } + if time.Now().After(deadline) { + return false, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(answerEvery): + } + } +} diff --git a/internal/bootstrap/retire_test.go b/internal/bootstrap/retire_test.go new file mode 100644 index 0000000..efdec44 --- /dev/null +++ b/internal/bootstrap/retire_test.go @@ -0,0 +1,165 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Retirement is destruction by omission, which is the host's ordinary behaviour: it owns what it +// applied and removes what it owns and is no longer declared. These tests defend the bundle +// surgery that expresses it, because a bundle that came out of it unparseable would be found by +// the apply — after the file on the machine had already been replaced. + +func produced(t *testing.T) []byte { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Bundle +} + +// The temporary control plane leaves the bundle, everything else stays, and what is left parses. +func TestTheTemporaryControlPlaneLeavesTheBundleAndNothingElseDoes(t *testing.T) { + before, err := declaration.ParseFileTrusted(produced(t)) + if err != nil { + t.Fatal(err) + } + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("the bundle without the control plane does not parse: %v\n%s", err, shorter) + } + if len(after.Resources) != len(before.Resources)-1 { + t.Fatalf("the bundle went from %d resources to %d, and one was removed", + len(before.Resources), len(after.Resources)) + } + if _, err := controlPlaneIn(after); err == nil { + t.Error("the bundle still declares a control plane") + } + // The store and the broker are still exactly what they were. A retirement that took the + // substrate with it would leave the machine with a module and nothing under it. + for id, name := range containerNames(before) { + if id == ControlPlaneID { + continue + } + if containerNames(after)[id] != name { + t.Errorf("%s was lost or renamed by the retirement", id) + } + } +} + +// **A `//` inside a string is not a comment.** The substrate's own bundle carries +// `postgres://…` several times, and a scanner that read the rest of those lines as a comment +// would lose braces and cut the wrong thing out — silently, because what it produced would still +// look like a file. +func TestASchemeInsideAStringIsNotReadAsAComment(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatal(err) + } + // The migration action, which is the resource holding the most `://` of anything here, is + // still whole. + found := false + for _, r := range after.Resources { + if r.Identity() == "context-schemas" { + found = true + } + } + if !found { + t.Error("the resource full of connection strings did not survive the removal") + } +} + +// Removing the FIRST element takes the comma after it rather than the comma before it, because +// there is no comma before it. Getting this wrong produces a leading comma, which is JSON nothing +// parses — and the file would already have been written. +func TestRemovingTheFirstResourceLeavesAValidList(t *testing.T) { + shorter, err := removeResource(produced(t), "container-runtime") + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("removing the first resource broke the bundle: %v\n%s", err, shorter) + } + for _, r := range after.Resources { + if r.Identity() == "container-runtime" { + t.Error("the first resource is still there") + } + } +} + +// A bundle that declares no such resource is refused rather than silently returned unchanged. A +// removal that removed nothing and reported success would leave the apply below with nothing to +// do and the installer claiming a pivot it did not finish. +func TestRemovingSomethingThatIsNotThereIsRefused(t *testing.T) { + if _, err := removeResource(produced(t), "nothing-of-the-sort"); err == nil { + t.Fatal("a bundle was reported to have had a resource removed that it never declared") + } +} + +// A re-run after the retirement finds a bundle with no control plane in it and says so, rather +// than failing. This is the idempotence of the last step, and it is the one a person is most +// likely to exercise: the pivot ends here, so a re-run to check ends here too. +func TestRetiringABundleThatAlreadyHasNoControlPlaneIsAlreadyDone(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + var said []string + out, err := RetireTheTemporaryControlPlane(context.Background(), Options{}, + nil, shorter, nil, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("a bundle with no control plane in it was not reported as already retired") + } + if !strings.Contains(strings.Join(said, "\n"), "already dropped") { + t.Errorf("the run does not say it was already done: %v", said) + } +} + +// A container the runtime still knows about after the apply is a pivot that did not finish. Two +// control planes on one mesh are both correct and neither is wrong — but the temporary one was +// supposed to go, and saying it went when it did not is the fault this project keeps naming. +func TestAContainerStillThereAfterRemovalIsNotGone(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + stillThere := &asked{answer: func(_ string, _ []string) (string, error) { + return "true running\n", nil + }} + gone, err := isGone(context.Background(), stillThere.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if gone { + t.Error("a container the runtime still describes was reported gone") + } + + removed := &asked{answer: func(_ string, _ []string) (string, error) { + return "", errors.New("No such object: temp-mesh-control") + }} + gone, err = isGone(context.Background(), removed.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if !gone { + t.Error("a container the runtime does not know about was not reported gone") + } +} diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go new file mode 100644 index 0000000..eab5a14 --- /dev/null +++ b/internal/bootstrap/talk.go @@ -0,0 +1,119 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// controlPlane is the running control plane, asked things. +// +// **Through `docker exec`, not over a network.** The control plane listens on nothing — `serve` is +// a broker consumer, and every administrative verb is a subcommand of the same binary that opens +// the stores directly (mesh-control's own usage). So the way to tell a mesh anything, from the +// machine the mesh is on, is to run its binary inside its own container. That is also what the lab +// does, and having the installer and the lab drive the mesh identically is the point: the lab is +// meant to exercise the installer, not a second procedure that resembles it. +// +// It carries which container, because the whole pivot turns on there being two of them: the +// substrate's `temp-mesh-control` for steps 6 to 9, and the module's `mesh-control` afterwards. +type controlPlane struct { + container string + run Runner + timeout time.Duration +} + +// within is the same control plane, asked with a different patience. +// +// A copy rather than a field somebody sets, so a slow command cannot leave every command after it +// slow: the caller that needs the long wait says so on the call. +func (c controlPlane) within(timeout time.Duration) controlPlane { + c.timeout = timeout + return c +} + +// tell runs a mesh-control subcommand and gives back what it said. +// +// The failure carries the command AND the output. A mesh-control refusal is a paragraph explaining +// what is wrong — "nothing provides route, wanted by registry" — and an installer that reported +// only "exit status 1" would throw away the one thing a person needs. +func (c controlPlane) tell(ctx context.Context, args ...string) (string, error) { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + out, err := c.run(asking, "docker", append( + []string{"exec", c.container, controlPlaneBinary}, args...)...) + if err != nil { + return out, fmt.Errorf("`%s %s` was refused: %w\n%s", + c.container, strings.Join(args, " "), err, indent(strings.TrimSpace(out))) + } + return out, nil +} + +// carry puts a file inside the control plane's container. +// +// **Because every command that takes a file reads it from its own filesystem.** `module add +// ` and `secret accept --from ` open a path, and the process doing the opening is +// inside the container. The installer is not. So the file is copied in first, exactly as the lab +// does it. +// +// The image is `FROM scratch` and has no shell, so nothing inside can move a file, change its mode +// or clean up after itself. What is copied in stays until the container is replaced — which, for +// the temporary control plane, is a container that gets removed at step 10 and takes its contents +// with it. +func (c controlPlane) carry(ctx context.Context, local, remote string) error { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + if _, err := c.run(asking, "docker", "cp", local, c.container+":"+remote); err != nil { + return fmt.Errorf("cannot put %s into %s at %s: %w", local, c.container, remote, err) + } + return nil +} + +// carrying writes bytes to a temporary file on the machine and copies them into the container. +// +// **0644, and that is not carelessness — 0600 would break it.** `docker cp` keeps the ownership and +// mode a file had outside, the control plane's image runs as 65534, and the process that reads +// these files is that one. The lab paid for this exactly once: a 0600 root-owned key copied in +// landed unreadable, `secret accept` failed with `permission denied`, and what depended on it +// crash-looped on material it never received. There is no shell in the image to chown it with. +// +// What goes through here is a module manifest and a store connection string. The connection is the +// same value the produced bundle already holds in the clear — a substrate names its own bootstrap +// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The +// file on the machine is removed at once, and the copy inside the container goes when the +// container does, which for the temporary control plane is step 10. +func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { + local := filepath.Join(os.TempDir(), name) + if err := os.WriteFile(local, content, 0o644); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.Remove(local) + return c.carry(ctx, local, remote) +} + +func indent(s string) string { + if s == "" { + return "" + } + return " " + strings.ReplaceAll(s, "\n", "\n ") +} + +// mentions reports whether one of a listing's lines starts with this exact word. +// +// Line-and-word rather than a substring search, because these listings are columns and a +// substring match would find `registry` inside `registry-mirror` and report a module installed +// that is not. Every one of mesh-control's `list` verbs prints the name first on the line. +func mentions(listing, name string) bool { + for _, line := range strings.Split(listing, "\n") { + first, _, _ := strings.Cut(strings.TrimSpace(line), " ") + if first == name { + return true + } + } + return false +} -- 2.54.0 From 0a88dc1f9d2e760310d0b40615cfdfc9ea7f69ee Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 00:03:14 +0200 Subject: [PATCH 6/9] bootstrap: follow the mount from the variable to the secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalogue's mesh-control manifest landed while this was being written, and it does what the ordinary case does: it keeps its secrets under /var/lib/mesh and mounts them into the container at /run/secrets, so MESH_STORE_INVENTORY_FILE names a path that no own-secret writes. Matching on the path alone found nothing and would have refused a correct manifest. So the lookup follows the volumes. It also reads the other shape the manifest uses — `VAR=${secret:name}` inside the environment file a container reads — which is how a value that is not a path gets in at all, and which is where the broker's two credentials live. That generalises what is delivered: every variable the module fills from a secret is looked up in the substrate's control plane. What the substrate names is accepted through `secret accept`; what it does not is left for the mesh to generate, and said so. A store connection the substrate does not name stays an error — a control plane that cannot open a context is not one. Checked against the real manifest (mesh-catalog feat/control-plane-module): five variables resolve, the placeholder pins in one place, and the container it waits for is `mesh-control`. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/bootstrap/control.go | 181 ++++++++++++++++++++--------- internal/bootstrap/control_test.go | 118 ++++++++++++++----- 2 files changed, 218 insertions(+), 81 deletions(-) diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index c406b53..6f6d937 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -218,26 +218,32 @@ func controlPlaneResourceIn(manifest []byte) string { // deliverStores carries the substrate's own database connections into the module. // // The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls -// these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a -// path, and the module's own-secret that writes that path is the secret to accept the connection -// as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the -// secret is called `inventory`, would seal a connection string under a name nothing reads and -// leave the mesh to invent random bytes for the one that is. +// these secrets is what is delivered. The installer does not guess that the secret holding the +// inventory connection is called `inventory`; it follows the manifest from the variable to the +// secret, and a manifest whose two ends do not meet is refused rather than half-delivered. +// +// **What is delivered is what the substrate already has, and only that.** The mesh generates an +// own-secret nobody supplied, which is right for something coming into existence and wrong for +// something that already exists. So every variable the module fills from a secret is looked up in +// the substrate's control plane: what it names is accepted, what it does not is left for the mesh +// to make. A store connection missing from the substrate is the one exception and is an error — +// a control plane that cannot open a context is not a control plane. func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, substrate *declaration.Declaration, say func(string)) ([]string, error) { - wanted, err := storeSecretsIn(manifest) + wanted, err := secretsByVariableIn(manifest) if err != nil { return nil, err } - if len(wanted) == 0 { + if !anyStoreIn(wanted) { return nil, fmt.Errorf( - "the %s module's manifest asks for no store connections. A control plane reaches each "+ - "context through its own credential (novox/hq ADR 0008), so a manifest naming none "+ - "describes a control plane that can open nothing.\n"+ + "the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+ + "each context through its own credential (novox/hq ADR 0008), so a manifest naming "+ + "none describes a control plane that can open nothing.\n"+ "The shape this installer delivers into is a file per context, named by an "+ - "own-secret, with %s%s in the container's environment pointing at it", - ControlPlaneModule, storeVariablePrefix, storeFileSuffix) + "own-secret, with %s%s pointing at it — directly, or at where that file is "+ + "mounted inside the container", + ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix) } temporary, err := controlPlaneIn(substrate) @@ -246,24 +252,29 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes } var delivered []string - for _, context := range sortedKeys(wanted) { - secret := wanted[context] - connection := temporary.Env[storeVariablePrefix+context] - if strings.TrimSpace(connection) == "" { - return delivered, fmt.Errorf( - "the %s module wants the %s store's connection and the bundle this installer "+ - "produced does not name one: its control plane has no %s.\n"+ - "These connections are the substrate's, created at genesis — the mesh cannot "+ - "invent them and the installer will not guess at one", - ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context) + for _, variable := range sortedKeys(wanted) { + secret := wanted[variable] + value := strings.TrimSpace(temporary.Env[variable]) + if value == "" { + if strings.HasPrefix(variable, storeVariablePrefix) { + return delivered, fmt.Errorf( + "the %s module wants %s and the bundle this installer produced does not name "+ + "one.\n"+ + "That connection is the substrate's, created at genesis — the mesh cannot "+ + "invent it and the installer will not guess at one", + ControlPlaneModule, variable) + } + // Not something the substrate made. The mesh generates its own, which is exactly what + // an own-secret is for; said so that nothing about the delivery is silent. + say(" the mesh will make " + secret + " — the substrate names no " + variable) + continue } // Into the container as a file, because `secret accept` reads a file or a prompt and the // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. - at := "/accepting-" + strings.ToLower(context) - if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context), - []byte(connection), at); err != nil { + at := "/accepting-" + secret + if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, @@ -271,60 +282,124 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes return delivered, err } delivered = append(delivered, secret) - say(" accepted " + secret + " — the " + strings.ToLower(context) + - " store, as the substrate made it") + say(" accepted " + secret + " — " + variable + ", as the substrate made it") } return delivered, nil } -// storeSecretsIn pairs each context with the secret its connection must be accepted as. +// secretsByVariableIn maps each environment variable the module fills from a secret to that +// secret's name. // -// Read out of the manifest twice over: the container's environment says which contexts are wanted -// and what file each expects, and the module's own-secrets say which secret writes which file. A -// pair that does not meet is refused rather than half-delivered. -func storeSecretsIn(manifest []byte) (map[string]string, error) { +// Two shapes, because the catalogue uses both: +// +// - `MESH_STORE__FILE` in the container's environment, naming a path the process reads. +// The path may be the own-secret's own path, or — more usually — where that file is mounted +// inside the container, in which case the volumes say which is which. Following the mount is +// not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at +// `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and +// refuse a correct manifest. +// - `VAR=${secret:name}` inside a file resource the container reads its environment from, which +// is how a value that is not a path gets in at all. +// +// A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and +// the process would find an empty file where a credential has to be, which presents as a container +// that will not start, a long way from the cause. +func secretsByVariableIn(manifest []byte) (map[string]string, error) { var m struct { OwnSecrets map[string]string `json:"own-secrets"` Resources []struct { - Type string `json:"type"` - Env map[string]string `json:"env"` + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + Volumes []string `json:"volumes"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) } - byPath := map[string]string{} + secretAt := map[string]string{} for name, path := range m.OwnSecrets { - byPath[path] = name + secretAt[path] = name } wanted := map[string]string{} for _, r := range m.Resources { - if r.Type != "container" { - continue - } - for key, path := range r.Env { - if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) { - continue + switch r.Type { + case "file": + for variable, secret := range secretsInContent(r.Content) { + wanted[variable] = secret } - context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix) - secret, ok := byPath[path] - if !ok { - return nil, fmt.Errorf( - "the %s module's container reads the %s store's connection from %s, and no "+ - "own-secret of that module writes that file.\n"+ - "So the mesh would seal nothing there and the control plane would find an "+ - "empty file where a connection string has to be. The manifest has to name "+ - "the two ends the same", - ControlPlaneModule, strings.ToLower(context), path) + case "container": + inside := mountedFrom(r.Volumes) + for key, path := range r.Env { + if !strings.HasPrefix(key, storeVariablePrefix) || + !strings.HasSuffix(key, storeFileSuffix) { + continue + } + on := path + if from, mounted := inside[path]; mounted { + on = from + } + secret, named := secretAt[on] + if !named { + return nil, fmt.Errorf( + "the %s module's container reads %s from %s, and no own-secret of that "+ + "module writes that file.\n"+ + "So the mesh would seal nothing there and the control plane would find "+ + "an empty file where a connection string has to be. The manifest has to "+ + "name the two ends the same, directly or through a mount", + ControlPlaneModule, key, path) + } + wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret } - wanted[context] = secret } } return wanted, nil } +// mountedFrom is where each path inside a container comes from outside it. +func mountedFrom(volumes []string) map[string]string { + inside := map[string]string{} + for _, volume := range volumes { + parts := strings.Split(volume, ":") + if len(parts) < 2 { + continue + } + inside[parts[1]] = parts[0] + } + return inside +} + +// secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment +// from. +func secretsInContent(content string) map[string]string { + found := map[string]string{} + for _, line := range strings.Split(content, "\n") { + variable, value, is := strings.Cut(strings.TrimSpace(line), "=") + if !is { + continue + } + const opens = "${secret:" + if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") { + continue + } + found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}") + } + return found +} + +// anyStoreIn reports whether any of these variables is a context's connection. +func anyStoreIn(wanted map[string]string) bool { + for variable := range wanted { + if strings.HasPrefix(variable, storeVariablePrefix) { + return true + } + } + return false +} + func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go index bf4d97c..9a6a5e1 100644 --- a/internal/bootstrap/control_test.go +++ b/internal/bootstrap/control_test.go @@ -11,28 +11,47 @@ import ( // that could go wrong quietly: pinning it to the wrong image, and delivering it store connections // the mesh invented rather than the ones the substrate actually made. -// theControlPlaneModule is the shape this installer codes against: one container using the -// catalogue's placeholder-digest convention, with each store connection delivered as a sealed -// secret written into a file and MESH_STORE__FILE pointing at it. +// theControlPlaneModule is the catalogue's manifest, trimmed to what this installer reads. +// +// A fixture rather than the file itself, unlike the substrate example the rewrite tests use: the +// catalogue is a different repository on a different branch, and a test that read it would pass or +// fail according to what somebody else had checked out. What it must stay faithful to is the +// SHAPE — the placeholder digest, the own-secret per context, the mount from the machine's path to +// the container's, and the environment file that fills what is not a path. const theControlPlaneModule = `{ "module": "mesh-control", "version": "1", + "slug": "control", "capabilities": ["container-runtime"], + "claims": [{"name": "the-control-plane", "scope": "mesh"}], "own-secrets": { - "inventory-store": "/var/lib/mesh/control/inventory", - "identity-store": "/var/lib/mesh/control/identity", - "licences-store": "/var/lib/mesh/control/licences" + "inventory": "/var/lib/mesh/mesh-control/inventory", + "identity": "/var/lib/mesh/mesh-control/identity", + "licences": "/var/lib/mesh/mesh-control/licences", + "broker": "/var/lib/mesh/mesh-control/broker", + "broker-management": "/var/lib/mesh/mesh-control/broker-management" }, "resources": [ - {"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, - {"id": "container", "type": "container", "name": "mesh-control", + {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, + {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-control/broker.env", + "mode": "0600", + "content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, + {"id": "server", "type": "container", "name": "mesh-control", "image": "mesh-control@` + placeholderDigest + `", "network": "host", "args": ["serve"], + "env-file": ["/var/lib/mesh/mesh-control/broker.env"], "env": { - "MESH_STORE_INVENTORY_FILE": "/var/lib/mesh/control/inventory", - "MESH_STORE_IDENTITY_FILE": "/var/lib/mesh/control/identity", - "MESH_STORE_LICENCES_FILE": "/var/lib/mesh/control/licences" - }} + "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", + "MESH_STORE_IDENTITY_FILE": "/run/secrets/identity", + "MESH_STORE_LICENCES_FILE": "/run/secrets/licences", + "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" + }, + "volumes": [ + "mesh-broker-tls:/broker-tls:ro", + "/var/lib/mesh/mesh-control/inventory:/run/secrets/inventory:ro", + "/var/lib/mesh/mesh-control/identity:/run/secrets/identity:ro", + "/var/lib/mesh/mesh-control/licences:/run/secrets/licences:ro" + ]} ] }` @@ -76,8 +95,8 @@ func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) { // otherwise be left half pinned, and fail inside an apply rather than here. func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { twice := strings.Replace(theControlPlaneModule, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"},`, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},`, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, {"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true, "image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1) @@ -99,22 +118,30 @@ func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { // context. The pairing is read from the manifest so that whatever the catalogue calls these // secrets is what is delivered. func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { - wanted, err := storeSecretsIn([]byte(theControlPlaneModule)) + wanted, err := secretsByVariableIn([]byte(theControlPlaneModule)) if err != nil { t.Fatal(err) } - for context, secret := range map[string]string{ - "INVENTORY": "inventory-store", - "IDENTITY": "identity-store", - "LICENCES": "licences-store", + // **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads + // them at /run/secrets. Matching on the path alone would find nothing and refuse a correct + // manifest, which is exactly the ordinary case in the catalogue. + for variable, secret := range map[string]string{ + "MESH_STORE_INVENTORY": "inventory", + "MESH_STORE_IDENTITY": "identity", + "MESH_STORE_LICENCES": "licences", + "MESH_BROKER_AMQP": "broker", + "MESH_BROKER_MANAGEMENT": "broker-management", } { - if wanted[context] != secret { - t.Errorf("the %s store's connection would be accepted as %q, want %q", - context, wanted[context], secret) + if wanted[variable] != secret { + t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret) } } + // And what the manifest fills from the machine rather than from a secret is left alone. + if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed { + t.Error("the address the mesh composes from the machine was treated as a secret") + } - // And the values are the substrate's own, taken from the produced bundle rather than composed. + // The values are the substrate's own, taken from the produced bundle rather than composed. rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) @@ -127,8 +154,9 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { if err != nil { t.Fatal(err) } - if len(delivered) != 3 { - t.Fatalf("%d connections were delivered, and the mesh holds three contexts: %v", + // Three stores and both halves of the broker: everything the substrate made and nothing else. + if len(delivered) != 5 { + t.Fatalf("%d values were delivered, and the substrate names five: %v", len(delivered), delivered) } for _, secret := range delivered { @@ -138,15 +166,49 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { } } +// A secret the substrate did not make is left for the mesh to make, and said so. Every other +// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh. +func TestASecretTheSubstrateNeverMadeIsLeftToTheMesh(t *testing.T) { + extra := strings.Replace(theControlPlaneModule, + `"broker": "/var/lib/mesh/mesh-control/broker",`, + `"broker": "/var/lib/mesh/mesh-control/broker", + "something-new": "/var/lib/mesh/mesh-control/something-new",`, 1) + extra = strings.Replace(extra, + `"content": "MESH_BROKER_AMQP=${secret:broker}\n`, + `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1) + + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + var said []string + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + for _, secret := range delivered { + if secret == "something-new" { + t.Error("a value the substrate never made was accepted as though it had") + } + } + if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") { + t.Errorf("nothing was said about the secret the mesh has to make: %v", said) + } +} + // A manifest whose container reads a file no own-secret writes is refused. The mesh would seal // nothing there and the control plane would find an empty file where a connection string has to // be — which presents as a control plane that will not start, three steps from the cause. func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { mismatched := strings.Replace(theControlPlaneModule, - `"inventory-store": "/var/lib/mesh/control/inventory"`, - `"inventory-store": "/var/lib/mesh/control/somewhere-else"`, 1) + `"inventory": "/var/lib/mesh/mesh-control/inventory",`, + `"inventory": "/var/lib/mesh/mesh-control/somewhere-else",`, 1) - _, err := storeSecretsIn([]byte(mismatched)) + _, err := secretsByVariableIn([]byte(mismatched)) if err == nil { t.Fatal("a manifest whose two ends do not meet was accepted") } -- 2.54.0 From cb5e137297d8ddece7c44ebb4a686df525431739 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 00:12:36 +0200 Subject: [PATCH 7/9] bootstrap: read the image id back from the runtime, never predict it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image id does not survive `docker save` -> transfer -> `docker load`. The id is the digest of the image's *configuration*, and a runtime rewrites that configuration as it loads: a newer Docker saves in one format, an older one stores it in another. Same layers, same program, different name. Measured on a live raise: saved on the workstation sha256:b86bb81ca2f9691f24f4725f50962d1e49c98c5ffe211113241243d42d18ceea loaded on the machine sha256:2dc219046c73702fc640317f0342a28ec962ef1e9ef547b2f02861c508ca78fb `internal/image`.ID read the id out of the carried tar and its comment said that was the id the runtime would assign. That is true on the machine the image was built on and false on every machine it is carried to — which is every machine this program exists for. The installer then either stopped at step 2 refusing the runtime's answer, or would have written a bundle naming an image the machine does not hold; and nothing serves an image named by the digest of its own configuration, which is the whole point of naming one that way, so the apply would have died inside a pull that cannot succeed. The lab hit this. So the image is identified by its TAG, which is ordinary metadata the tar carries through unchanged. The runtime is asked what that tag resolves to before the load (already held, nothing to do) and again after (this is what the bundle names). The tag never reaches the bundle — a pinned bundle may not rely on one, ADR 0006 — it is how the id is obtained, not what is written down. - image.ID becomes image.ArchiveID, and says plainly that it is a fact about the file and not a prediction about any machine. It is kept for reports, and printed beside the runtime's answer whenever the two differ. - Idempotence is decided from what the runtime holds under the tag, not from a predicted id, which cannot answer the question at all here. - An untagged archive is refused, in preflight and again at the load: there would be no portable name to ask about, and the only thing left is scraping a sentence `docker load` writes for a person. `make bootstrap` refuses an id or an untagged image, so it is caught in front of whoever can fix it. - A dry run cannot know the id and says so rather than pretending. Run refuses to write a bundle carrying an unconfirmed id at all. Tests: the injected Runner now answers with an id DIFFERING from the tar's, and the runtime's answer is what must be used. The test that refused a differing id encoded the mistake and is replaced by one refusing an answer that is not an id at all. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- Makefile | 9 ++ internal/bootstrap/bootstrap.go | 44 ++++++- internal/bootstrap/load.go | 165 ++++++++++++++++------- internal/bootstrap/load_test.go | 226 +++++++++++++++++++++++--------- internal/bootstrap/preflight.go | 21 ++- internal/image/image.go | 49 ++++--- internal/image/image_test.go | 38 +++--- 7 files changed, 410 insertions(+), 142 deletions(-) diff --git a/Makefile b/Makefile index d5bf716..1c5f2c2 100644 --- a/Makefile +++ b/Makefile @@ -68,9 +68,18 @@ host: # # The saved image occupies the embed slot for the length of one build and the placeholder goes # back, exactly as `host:` does with the bundle. Nothing large is ever committed. +# +# IMAGE must be a NAME:TAG and not an id. The installer identifies the carried image by its tag, +# because an image id is the digest of the image's configuration and a runtime REWRITES that +# configuration as it loads — so the id in the archive is not the id the receiving machine will +# hold, and the tag is the only name that survives the transfer. Saving by id produces an archive +# with no tags at all, which the installer refuses; caught here instead, in front of the person who +# can fix it. bootstrap: @test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; } + @case "$(IMAGE)" in sha256:*) echo "IMAGE=$(IMAGE) is an image id. The installer identifies the carried image by its tag, because an id is the digest of a configuration that a runtime rewrites as it loads. Pass a name:tag"; exit 1;; esac @docker image inspect "$(IMAGE)" >/dev/null 2>&1 || { echo "this machine does not hold $(IMAGE) — build it in mesh-control with 'make image'"; exit 1; } + @test -n "$$(docker image inspect --format '{{len .RepoTags}}' "$(IMAGE)" | grep -v '^0$$')" || { echo "$(IMAGE) has no repository tag, so the saved archive would carry no name the installer can ask a runtime about. Tag it first: docker tag $(IMAGE) mesh-control:"; exit 1; } @cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder @docker save --output internal/image/control-plane.tar "$(IMAGE)" @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o mesh-bootstrap ./cmd/mesh-bootstrap; \ diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 1771aee..ae31339 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -20,6 +20,13 @@ // configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq // ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream // `name@sha256:` references and are pulled from the internet like anything else. +// +// **And that id is read back from the machine, never predicted from the archive.** The digest of a +// configuration is not portable: a runtime rewrites the configuration as it loads, so the same +// bytes are held under a different name on the machine that receives them than on the one that +// saved them. The archive is identified by its TAG, which does survive the transfer, and the id +// the bundle names is whatever the runtime answers for that tag afterwards. See Load, which +// carries the measurement. package bootstrap import ( @@ -151,12 +158,22 @@ type Result struct { System string `json:"system"` DryRun bool `json:"dry-run,omitempty"` - // Image is the control plane's image id — what the produced bundle names it by. + // Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and + // what the produced bundle names it by. Image string `json:"image,omitempty"` - // ImageTags is what that image was called when it was saved. Decoration, for a person. + // ImageArchive is what the carried archive calls the same image. Reported because it is + // routinely a DIFFERENT id: a runtime rewrites an image's configuration as it loads, and an id + // is that configuration's digest. Never what the bundle names. + ImageArchive string `json:"image-in-archive,omitempty"` + // ImageTag is the name the runtime was asked by, which is how the id above was obtained. + ImageTag string `json:"image-tag,omitempty"` + // ImageTags is everything the image was called when it was saved. ImageTags []string `json:"image-tags,omitempty"` // ImageHeld is true when the machine already held it and nothing was loaded. ImageHeld bool `json:"image-already-held,omitempty"` + // ImagePredicted is true when Image is the archive's id because nothing was loaded — a dry run + // only, and the reason a dry run does not claim to know what would be applied. + ImagePredicted bool `json:"image-id-is-a-prediction,omitempty"` // Bundle is where the produced bundle was written, and what was done to produce it. Bundle string `json:"bundle,omitempty"` @@ -276,6 +293,8 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepLoad, err) } result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held + result.ImageArchive, result.ImageTag = loaded.Archive, loaded.Tag + result.ImagePredicted = loaded.Predicted // ---- 3. bundle ---------------------------------------------------------------------- say("bundle — what this machine will be asked to be") @@ -301,6 +320,10 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro say(fmt.Sprintf(" its image %s — the template already named it, nothing rewritten", rewritten.Now)) } + if loaded.Predicted { + say(" UNCONFIRMED that id is the archive's and no runtime has been asked. A real " + + "run reads it back.") + } for _, kept := range rewritten.Kept { say(" left alone " + kept) } @@ -330,10 +353,27 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + "loaded or applied" + if loaded.Predicted { + result.Stopped += ". The image id in it is the archive's own and is not necessarily " + + "the one this machine would hold — a runtime rewrites an image's configuration as " + + "it loads, and the id is that configuration's digest" + } say("\n" + result.Stopped) return result, nil } + // **Nothing predicted is ever written down.** The line above is the only path on which + // `loaded.ID` can be the archive's id, and it returns. Asserted here rather than left to the + // reader, because what would follow is a bundle naming an image this machine does not hold — + // and nothing serves an image named by the digest of its own configuration, so it would fail + // inside a pull that cannot succeed, three steps from the cause. + if loaded.Predicted { + return result, failed(StepBundle, fmt.Errorf( + "the control plane's image id was never confirmed against this machine's runtime, and "+ + "the bundle was about to be written with it. This is a fault in the installer, not "+ + "in the machine")) + } + if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil { return result, failed(StepBundle, err) } diff --git a/internal/bootstrap/load.go b/internal/bootstrap/load.go index a44db70..4e1fc71 100644 --- a/internal/bootstrap/load.go +++ b/internal/bootstrap/load.go @@ -11,26 +11,54 @@ import ( // Loaded is the control plane's image on this machine. type Loaded struct { - // ID is what the bundle will name the image by: sha256 of its own configuration. + // ID is what THIS RUNTIME holds the image as, read back from it after the load. It is what the + // bundle names, and outside a dry run it is never a prediction — see Load. ID string - // Tags is what it was called when it was saved. For a person, never for the bundle. + // Archive is what the carried tar calls the same image. Kept because the two differ in + // practice, and a report showing only one of them cannot say that they did. Never what the + // bundle names. + Archive string + // Tag is the name the runtime is asked by. Load-bearing rather than decoration: it is the one + // name that survives `docker save` and `docker load` unchanged. + Tag string + // Tags is everything the archive was called when it was saved. Tags []string - // Held is true when the machine already had it and nothing was loaded. + // Held is true when the machine already held it and nothing moved. Held bool + // Predicted is true only on a dry run, where nothing was loaded and ID is therefore the + // archive's id — which is not necessarily the one this machine would end up with. + Predicted bool } -// Load puts the carried control-plane image into this machine's container runtime. +// Load puts the carried control-plane image into this machine's container runtime, and reports +// what the runtime decided to call it. // -// **Idempotent by asking first, which is possible because the id is a fact about the file.** The -// image id is read out of the saved tar (see `internal/image`.ID) before the runtime is asked -// anything, so this can ask "do you already hold exactly this image" — and on the second, third -// and tenth run of the installer the answer is yes and nothing is loaded. A load that scraped the -// id out of what `docker load` printed could only know that after loading, so it would load every -// time and report the same thing either way. +// **The digest of a configuration is not portable across runtimes, and that is why the id is read +// back rather than predicted.** An image id is the sha256 of the image's configuration document, +// and a runtime REWRITES that document as it loads: a newer Docker saves in one format, an older +// one stores it in another, and the same layers come out under a different name. Measured on a +// live raise, an image saved as `sha256:b86bb81c…` on a workstation was loaded as +// `sha256:2dc21904…` on the machine it was carried to. +// +// This code used to read the id out of the tar before the runtime was asked anything and use it +// for both idempotence and the bundle. That is right on the machine the image was built on and +// wrong on every machine it is carried to — which is every machine this program exists for. The +// bundle would have named an image the machine does not hold; nothing serves an image named by +// the digest of its own configuration, which is the whole point of naming one that way; and the +// apply would have stopped inside a pull that cannot succeed. The lab hit exactly this. +// +// **So the image is identified by its TAG.** A tag is ordinary metadata the tar carries through +// unchanged, and asking the runtime what a tag resolves to is asking the only party entitled to +// answer. The tag never reaches the bundle — a pinned bundle may not rely on one +// (novox/hq ADR 0006) — it is how the id is obtained, not what is written down. +// +// **Idempotence is decided from what the runtime holds.** The tag is asked before the load and +// again after: the same id either side means nothing moved, which is a fact about this machine +// rather than a guess about the file. A tag that already resolves means the image is already +// held, and nothing is loaded at all. // // It reads back (novox/hq ADR 0018). A load that reported success and left nothing there is a -// failure, not a convergence, and the apply would then meet a bundle naming an image the machine -// does not hold — which fails correctly but two steps too late. +// failure, not a convergence. func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loaded, error) { saved, err := image.Saved() if err != nil { @@ -42,22 +70,50 @@ func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loade // loadImage is Load with the carried bytes handed in, so the whole path can be tested against a // saved image a test builds rather than against whatever a particular build embedded. func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say func(string)) (Loaded, error) { - id, err := image.ID(saved) + archiveID, err := image.ArchiveID(saved) if err != nil { return Loaded{}, err } - loaded := Loaded{ID: id, Tags: image.Tags(saved)} + loaded := Loaded{Archive: archiveID, Tags: image.Tags(saved)} - if held, err := holdsImage(ctx, run, id); err != nil { + // **An untagged archive is a build-time fault, refused here rather than worked around.** + // Without a tag there is no portable name to ask the runtime about, and the only thing left is + // scraping the sentence `docker load` prints for a person — which differs between runtime + // versions and is exactly the kind of guess this whole step exists to stop making. The release + // target tags the image; an installer built without one was built wrong. + loaded.Tag = firstOr(loaded.Tags, "") + if loaded.Tag == "" { + return loaded, fmt.Errorf( + "the carried control-plane image has no tag, so there is no portable name to ask this "+ + "machine's runtime what id it gave it.\n"+ + "An image id is the digest of the image's configuration and a runtime rewrites "+ + "that as it loads, so the id in the archive (%s) is not necessarily the id this "+ + "machine will hold — and a bundle naming the wrong one names an image nothing "+ + "serves. Rebuild the installer with a tagged image: `make bootstrap "+ + "IMAGE=:`", archiveID) + } + + // Already held? Asked of the runtime, by the tag, before anything is written anywhere. + before, err := idOfImage(ctx, run, loaded.Tag) + if err != nil { return loaded, err - } else if held { - loaded.Held = true - say(" already held " + id + " — nothing loaded") + } + if before != "" { + loaded.ID, loaded.Held = before, true + say(" already held " + before + " as " + loaded.Tag + " — nothing loaded") + sayIfDifferent(say, archiveID, before) return loaded, nil } if dryRun { - say(fmt.Sprintf(" would load %s (%d bytes)", id, len(saved))) + // Nothing is loaded, so the runtime has not been asked to decide anything — and what it + // would decide cannot be worked out from here. Said, rather than quietly guessed at. + loaded.ID, loaded.Predicted = archiveID, true + say(fmt.Sprintf(" would load %s (%d bytes) as %s", archiveID, len(saved), loaded.Tag)) + say(" NOT THE FINAL ID a runtime rewrites an image's configuration as it loads, and an") + say(" id is that configuration's digest. The bundle names what the") + say(" runtime answers for " + loaded.Tag + " afterwards, which a dry run") + say(" cannot ask for without loading.") return loaded, nil } @@ -83,40 +139,61 @@ func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say f "the container runtime would not load the carried control-plane image: %w", err) } - // Read back. This is what makes "loaded" a fact rather than an intention. - held, err := holdsImage(ctx, run, id) + // Read back, and THIS is the answer the bundle is rewritten to. + after, err := idOfImage(ctx, run, loaded.Tag) if err != nil { return loaded, err } - if !held { + if after == "" { return loaded, fmt.Errorf( - "the load reported success and this machine does not hold %s.\n"+ - "The bundle names the control plane by that id and nothing serves it, so the "+ - "apply would refuse. Check what `docker load` actually took", id) + "the load reported success and this machine holds nothing called %s.\n"+ + "The bundle names the control plane by the id this runtime assigned, so there is "+ + "nothing to name. Check what `docker load` actually took", loaded.Tag) } - say(" loaded " + id) + loaded.ID = after + say(" loaded " + after + " as " + loaded.Tag) + sayIfDifferent(say, archiveID, after) return loaded, nil } -// holdsImage asks the runtime whether this exact image is present. +// sayIfDifferent reports the archive's own id when the runtime chose another. // -// It asks for the id back rather than reading the exit code, because an image inspected by id and -// an image inspected by a tag that happens to point somewhere else are the same successful -// command. What is wanted is "this one", and the answer says which one. -func holdsImage(ctx context.Context, run Runner, id string) (bool, error) { - out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", id) - if err != nil { - // Absent is an answer, not a failure. Every other reason the runtime might refuse looks - // the same from here — which is why preflight proves the runtime answers before this runs, - // rather than this trying to tell the two apart from an exit code. - return false, nil +// Said every time it happens, because it is surprising, it is ordinary, and somebody comparing +// this report against `docker images` on the machine the image was built on would otherwise +// conclude that the wrong image had been carried. +func sayIfDifferent(say func(string), archiveID, held string) { + if archiveID == held { + return } - got := strings.TrimSpace(out) - if got != id { - return false, fmt.Errorf( - "asked for image %s, the runtime answered %q. An image id is the digest of the "+ - "image's own configuration, so these are two different images and the bundle "+ - "would name the wrong one", id, got) + say(" the archive says " + archiveID) + say(" this runtime stored the same image under a different configuration, " + + "which is ordinary — the bundle names what the machine holds") +} + +// idOfImage asks the runtime what it holds under a name, or empty if it holds nothing. +// +// It asks for the id back rather than reading the exit code, because the id is what is wanted and +// an exit code is not it. Absent is an answer and not a failure: every other reason the runtime +// might refuse looks the same from here, which is why preflight proves the runtime answers before +// this runs rather than this trying to tell the two apart from an exit status. +// +// What it will not do is accept an answer that is not an image id. That answer becomes the name +// the bundle applies on a machine with no mesh to check anything against, so it is checked here +// where the refusal can say whose mistake it is. +func idOfImage(ctx context.Context, run Runner, name string) (string, error) { + out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", name) + if err != nil { + return "", nil } - return true, nil + got := strings.TrimSpace(firstLineOf(out)) + if got == "" { + return "", nil + } + if !isImageID(got) { + return "", fmt.Errorf( + "asked what this machine holds as %q, the runtime answered %q, which is not an image "+ + "id. The bundle would name the control plane by that answer, and it is refused "+ + "rather than written down", name, got) + } + return got, nil } diff --git a/internal/bootstrap/load_test.go b/internal/bootstrap/load_test.go index 0442875..b37fdc2 100644 --- a/internal/bootstrap/load_test.go +++ b/internal/bootstrap/load_test.go @@ -41,12 +41,17 @@ func (a *asked) ran(fragment string) bool { return false } -func savedImageFixture(t *testing.T, digest string) []byte { +// savedImageFixture builds what `docker save` produces, tagged `mesh-control:test` unless a test +// asks for something else. Pass no tags for an archive saved without one. +func savedImageFixture(t *testing.T, digest string, tags ...string) []byte { t.Helper() + if tags == nil { + tags = []string{"mesh-control:test"} + } entries, err := json.Marshal([]struct { Config string RepoTags []string - }{{Config: digest + ".json", RepoTags: []string{"mesh-control:test"}}}) + }{{Config: digest + ".json", RepoTags: tags}}) if err != nil { t.Fatal(err) } @@ -67,17 +72,83 @@ func savedImageFixture(t *testing.T, digest string) []byte { return buffer.Bytes() } -const fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333" +// fixtureDigest is what the ARCHIVE calls the image, and runtimeDigest is what a runtime calls it +// after loading the same bytes. They differ on purpose, because they differ in reality: an image +// id is the digest of the image's configuration, and a runtime rewrites that configuration as it +// loads. Measured on a live raise, `mesh-control:development` was `sha256:b86bb81c…` on the +// workstation that saved it and `sha256:2dc21904…` on the machine that loaded it. +const ( + fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333" + runtimeDigest = "4444444444444444444444444444444444444444444444444444444444444444" +) + +// **The id the bundle is named by comes from the RUNTIME, not from the archive.** +// +// This is the test for the fault that took the lab down. The installer used to read the id out of +// the carried tar and use it for the bundle, which is correct on the machine the image was built +// on and wrong on every machine it is carried to — and a bundle naming an id the machine does not +// hold names an image nothing can serve, because an image named by the digest of its own +// configuration is by definition served by nobody. The apply then stops inside a pull that cannot +// succeed, three steps from the cause. +// +// So the image is identified by its TAG, which survives save and load unchanged, and the runtime +// is asked what that tag resolves to. +func TestTheIdComesFromTheRuntimeAndNotFromTheArchive(t *testing.T) { + runtime := &asked{} + inspected := 0 + runtime.answer = func(_ string, args []string) (string, error) { + switch { + case len(args) > 1 && args[0] == "image" && args[1] == "inspect": + inspected++ + if inspected == 1 { + // Nothing held yet. + return "", errors.New("Error: No such image") + } + // Loaded — and stored under a configuration of the runtime's own making. + return "sha256:" + runtimeDigest + "\n", nil + case len(args) > 0 && args[0] == "load": + return "Loaded image: mesh-control:test\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + } + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the bundle would name %q; this machine holds sha256:%s", loaded.ID, runtimeDigest) + } + if loaded.Archive != "sha256:"+fixtureDigest { + t.Errorf("the archive's own id is reported as %q", loaded.Archive) + } + if loaded.Predicted { + t.Error("a real run reported its id as a prediction") + } + // The runtime was asked BY THE TAG, which is the only name that survives the transfer. + if !runtime.ran("docker image inspect --format {{.Id}} mesh-control:test") { + t.Errorf("the runtime was never asked what the tag resolves to: %v", runtime.commands) + } + // And the difference is said out loud, or somebody comparing this against `docker images` on + // the build machine concludes the wrong image was carried. + if !strings.Contains(strings.Join(said, "\n"), "the archive says") { + t.Errorf("nothing was said about the two ids differing: %v", said) + } +} // A machine that already holds the image is not loaded again, and says so. // // This is the idempotence the installer's usefulness rests on: it is run over and over while // somebody gets a machine working, and a step that did its work again every time would be -// indistinguishable from one that had never run. +// indistinguishable from one that had never run. **Decided from what the runtime holds under the +// tag, not from what the archive predicts** — a predicted id cannot answer this question at all on +// a machine whose runtime rewrites configurations. func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { runtime := &asked{answer: func(_ string, args []string) (string, error) { if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { - return "sha256:" + fixtureDigest + "\n", nil + return "sha256:" + runtimeDigest + "\n", nil } return "", fmt.Errorf("unexpected command: %v", args) }} @@ -92,6 +163,10 @@ func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { if !loaded.Held { t.Error("the machine already held the image and the load did not say so") } + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the id reported for an already-held image is %q, and the machine holds sha256:%s", + loaded.ID, runtimeDigest) + } if runtime.ran("docker load") { t.Errorf("the image was loaded again although the machine held it: %v", runtime.commands) } @@ -100,45 +175,6 @@ func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { } } -// The id comes out of the file, and the bundle is named by it. -// -// Not scraped from what `docker load` prints — that is a sentence for a person, which reads -// `Loaded image: name:tag` or `Loaded image ID: sha256:…` depending on how the image was saved. -// A program depending on which one a runtime chose would be depending on a runtime version. -func TestTheImageIdComesFromTheCarriedFileNotFromWhatTheRuntimeSays(t *testing.T) { - runtime := &asked{} - inspected := 0 - runtime.answer = func(_ string, args []string) (string, error) { - switch { - case len(args) > 1 && args[0] == "image" && args[1] == "inspect": - inspected++ - if inspected == 1 { - return "", errors.New("Error: No such image") - } - return "sha256:" + fixtureDigest + "\n", nil - case len(args) > 0 && args[0] == "load": - // Deliberately says something else entirely. The id must not come from here. - return "Loaded image: some-other-name:whatever\n", nil - } - return "", fmt.Errorf("unexpected command: %v", args) - } - - loaded, err := loadImage(context.Background(), runtime.run, - savedImageFixture(t, fixtureDigest), false, func(string) {}) - if err != nil { - t.Fatal(err) - } - if loaded.ID != "sha256:"+fixtureDigest { - t.Errorf("the image id is %q, want sha256:%s", loaded.ID, fixtureDigest) - } - if loaded.Held { - t.Error("an image that had to be loaded was reported as already held") - } - if !runtime.ran("docker load") { - t.Errorf("the image was never loaded: %v", runtime.commands) - } -} - // A load that reported success and left nothing there is a failure, not a convergence // (novox/hq ADR 0018). Without the read-back it would surface later as the host refusing a bundle // naming an image nothing serves — a true message about the wrong thing. @@ -155,14 +191,17 @@ func TestALoadThatLeftNothingBehindIsAFailure(t *testing.T) { if err == nil { t.Fatal("a load that left nothing on the machine was reported as success") } - if !strings.Contains(err.Error(), fixtureDigest) { - t.Errorf("the failure does not say which image is missing: %v", err) + // Named by the tag, because that is what was asked about and what is missing. + if !strings.Contains(err.Error(), "mesh-control:test") { + t.Errorf("the failure does not say what this machine holds nothing of: %v", err) } } -// A dry run changes nothing, and still knows the id — because the id is a property of the carried -// file. That is what lets `--dry-run` produce and check the real bundle rather than a guess. -func TestADryRunLearnsTheIdAndLoadsNothing(t *testing.T) { +// **A dry run cannot know the id, and says so rather than pretending.** It loads nothing, so no +// runtime has decided anything, and the id in the archive is a fact about a file rather than a +// prediction about this machine. `--dry-run` still produces and checks the bundle's shape; what it +// cannot promise is the one value that only a load can settle. +func TestADryRunLoadsNothingAndSaysTheIdIsUnconfirmed(t *testing.T) { runtime := &asked{answer: func(_ string, args []string) (string, error) { if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { return "", errors.New("Error: No such image") @@ -170,16 +209,68 @@ func TestADryRunLearnsTheIdAndLoadsNothing(t *testing.T) { return "", fmt.Errorf("a dry run ran %v", args) }} + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), true, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if !loaded.Predicted { + t.Error("a dry run reported an id no runtime had confirmed as though it had been") + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("a dry run reported %q, and the archive says sha256:%s", loaded.ID, fixtureDigest) + } + if runtime.ran("docker load") { + t.Errorf("a dry run loaded an image: %v", runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "NOT THE FINAL ID") { + t.Errorf("a dry run did not say its id is unconfirmed: %v", said) + } +} + +// A dry run on a machine that already holds the image DOES know the id, because the runtime was +// asked and answered. Reading is not changing, so a dry run is entitled to that. +func TestADryRunOnAMachineThatHoldsItKnowsTheRealId(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "sha256:" + runtimeDigest + "\n", nil + } + return "", fmt.Errorf("a dry run ran %v", args) + }} + loaded, err := loadImage(context.Background(), runtime.run, savedImageFixture(t, fixtureDigest), true, func(string) {}) if err != nil { t.Fatal(err) } - if loaded.ID != "sha256:"+fixtureDigest { - t.Errorf("a dry run did not work out the image id: %q", loaded.ID) + if loaded.Predicted { + t.Error("an id this machine's runtime supplied was reported as a prediction") } - if runtime.ran("docker load") { - t.Errorf("a dry run loaded an image: %v", runtime.commands) + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the id is %q, and the runtime said sha256:%s", loaded.ID, runtimeDigest) + } +} + +// **An untagged archive is a build-time fault, refused rather than worked around.** Without a tag +// there is no portable name to ask the runtime about, and the only thing left is scraping the +// sentence `docker load` prints for a person — which differs between runtime versions and is +// exactly the kind of guess this whole step exists to stop making. +func TestAnUntaggedArchiveIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + return "", fmt.Errorf("nothing should have been run: %v", args) + }} + + _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest, []string{}...), false, func(string) {}) + if err == nil { + t.Fatal("an archive with no tag was accepted, and there is no way to ask about it") + } + if !strings.Contains(err.Error(), "make bootstrap") { + t.Errorf("the refusal does not say how to build one that is tagged: %v", err) + } + if len(runtime.commands) != 0 { + t.Errorf("the machine was touched first: %v", runtime.commands) } } @@ -198,14 +289,25 @@ func TestAnInstallerCarryingNoImageSaysSoRatherThanRaisingHalfAMesh(t *testing.T } } -// Two different images cannot share an id, so an answer that is not the id asked about means the -// runtime is talking about something else. Reported rather than believed. -func TestARuntimeAnsweringAboutADifferentImageIsRefused(t *testing.T) { - runtime := &asked{answer: func(_ string, _ []string) (string, error) { - return "sha256:" + strings.Repeat("9", 64) + "\n", nil - }} - if _, err := loadImage(context.Background(), runtime.run, - savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil { - t.Fatal("the runtime answered about a different image and it was accepted") +// An answer that is not an image id is refused rather than written into a bundle. +// +// **This replaces a test that refused an answer differing from the archive's id.** That test +// encoded the mistake: a differing id is now the expected case, not a fault, because a runtime +// rewrites an image's configuration as it loads. What is still worth refusing is an answer that is +// not an id at all — that value becomes the name a bundle applies on a machine with no mesh to +// check anything against, so it is checked where the refusal can say whose mistake it is. +func TestARuntimeAnsweringSomethingThatIsNotAnImageIdIsRefused(t *testing.T) { + for _, nonsense := range []string{ + "mesh-control:test", + "sha256:" + strings.Repeat("9", 63), + "", + } { + runtime := &asked{answer: func(_ string, _ []string) (string, error) { + return nonsense + "\n", nil + }} + if _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil { + t.Errorf("the runtime answered %q and it was accepted as an image id", nonsense) + } } } diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index 476a530..e13e276 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -36,12 +36,27 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte if err != nil { return nil, err } - carriedID, err := image.ID(saved) + // + // The id said here is the ARCHIVE's, and it is reported as such: it is a fact about the file + // and not about this machine. What this runtime will call the image once it holds it is the + // runtime's decision, made at the load, and asked for there (see Load). + carriedID, err := image.ArchiveID(saved) if err != nil { return nil, err } - say(fmt.Sprintf(" control plane %s carried (%s)", - firstOr(image.Tags(saved), "untagged"), carriedID)) + tag := firstOr(image.Tags(saved), "") + if tag == "" { + // Refused here as well as at the load, because preflight's whole job is to find at the + // start what would otherwise be found half way through — and this would be found after an + // image had been written to disk and handed to a container runtime. + return nil, fmt.Errorf( + "the carried control-plane image has no tag, and the installer identifies it by one: "+ + "an image id is the digest of a configuration that a runtime rewrites as it loads, "+ + "so the archive's id (%s) is not necessarily the id this machine would hold.\n"+ + "Rebuild the installer with a tagged image: `make bootstrap IMAGE=:`", + carriedID) + } + say(fmt.Sprintf(" control plane %s carried (the archive calls it %s)", tag, carriedID)) // 2. Is the template there, and is it a substrate? template, err := os.ReadFile(o.Template) diff --git a/internal/image/image.go b/internal/image/image.go index 5e94196..55d77c5 100644 --- a/internal/image/image.go +++ b/internal/image/image.go @@ -82,20 +82,33 @@ type manifestEntry struct { RepoTags []string `json:"RepoTags"` } -// ID is the image id the runtime will give this image once it is loaded, read out of the tar. +// ArchiveID is what this archive calls the image it holds. **It is not necessarily the id the +// runtime will assign when the archive is loaded, and it must never be what a bundle names.** // -// **Read here rather than parsed out of what `docker load` prints.** The load prints a sentence -// for a person — `Loaded image: name:tag` or `Loaded image ID: sha256:…`, depending on whether the -// image was saved with a tag — and a program that scraped it would be depending on which of those -// a particular runtime version chose. The id is a fact about the file, available before the -// runtime is asked anything, which is also what makes the load idempotent: the installer can ask -// whether the machine already holds THIS image before loading it. +// This was called ID, and its comment said it was the id the runtime would give the image once +// loaded. That was wrong, and wrong in the worst available way: right on the machine the image +// was saved on, and wrong on the machine it was carried to. An image id is the sha256 of the +// image's *configuration document*, and a runtime rewrites that document as it loads — a newer +// Docker saves in one format and an older one stores it in another. Same layers, same program, +// different name. Measured on a live raise: // -// An image id is the sha256 of the image's configuration document (novox/hq ADR 0006, and see -// `internal/declaration`'s checkImage). `manifest.json` names that document by its digest — as -// `<64hex>.json` in the older layout and `blobs/sha256/<64hex>` in the OCI one — so both forms -// reduce to the same sixty-four characters. -func ID(saved []byte) (string, error) { +// saved on the workstation sha256:b86bb81ca2f9691f24f4725f50962d1e49c98c5ffe211113241243d42d18ceea +// loaded on the machine sha256:2dc219046c73702fc640317f0342a28ec962ef1e9ef547b2f02861c508ca78fb +// +// A bundle naming this id would then name an image the machine does not hold — and nothing serves +// an image named by the digest of its own configuration, which is the whole point of naming one +// that way, so the apply would stop at a pull that cannot succeed. The id a bundle names is read +// back from the runtime after the load (`internal/bootstrap`.Load), which is the only place it is +// a fact rather than a prediction. +// +// What it is still good for is a statement about the FILE — which build somebody embedded — and +// as a hint printed beside the runtime's answer when the two differ, so a person can see that +// they did. +// +// `manifest.json` names the configuration document by its digest — as `<64hex>.json` in the older +// layout and `blobs/sha256/<64hex>` in the OCI one — so both forms reduce to the same sixty-four +// characters. +func ArchiveID(saved []byte) (string, error) { manifest, err := fileFromTar(saved, "manifest.json") if err != nil { return "", err @@ -125,11 +138,15 @@ func ID(saved []byte) (string, error) { return "sha256:" + digest, nil } -// Tags is what the saved image was called when it was saved, for a person reading a report. +// Tags is what the saved image was called when it was saved. // -// Decoration, and said so: the installer names the image by its id everywhere it matters, because -// a tag is exactly what a pinned bundle may not rely on (novox/hq ADR 0006). This is here so a -// report can say which build somebody embedded, which is otherwise sixty-four characters of hex. +// **Not decoration any more, and this comment used to say it was.** A tag is exactly what a pinned +// bundle may not rely on (novox/hq ADR 0006) and none of this ever reaches a bundle — but the tag +// is how the installer ASKS the runtime what id it assigned, because the id itself is not +// knowable beforehand (see ArchiveID). It is the one name that survives `docker save` and +// `docker load` unchanged, which is precisely what the id does not. +// +// It also still says which build somebody embedded, which is otherwise sixty-four hex characters. func Tags(saved []byte) []string { manifest, err := fileFromTar(saved, "manifest.json") if err != nil { diff --git a/internal/image/image_test.go b/internal/image/image_test.go index 74970db..ef93e6a 100644 --- a/internal/image/image_test.go +++ b/internal/image/image_test.go @@ -39,20 +39,25 @@ func manifest(t *testing.T, config string, tags ...string) string { return string(raw) } -// The image id is read from the FILE, before any runtime is asked anything. +// The archive's own id is read from the FILE, in both layouts `docker save` has used. // -// That is what makes the load idempotent: knowing the id in advance lets the installer ask "do you -// already hold exactly this" instead of loading and then finding out. Scraping it from what -// `docker load` prints would only be possible after loading, so the second run of an installer -// would load again every time and be unable to say it had not. -func TestTheImageIdIsReadFromTheSavedFile(t *testing.T) { +// **This test used to say that reading it here was what made the load idempotent — that knowing +// the id in advance let the installer ask "do you already hold exactly this". That was wrong.** An +// id is the digest of the image's configuration document, and a runtime rewrites that document as +// it loads, so this is a fact about the archive and not a prediction about any machine. What the +// installer asks a runtime by is the TAG, and what a bundle names is the answer the runtime gives +// back (`internal/bootstrap`.Load). +// +// It is still read and still checked, because it is what says which build somebody embedded — and +// because printing it beside the runtime's answer is how a person sees that the two differ. +func TestTheArchivesOwnIdIsReadFromTheSavedFile(t *testing.T) { digest := strings.Repeat("a", 64) // Both layouts `docker save` has used. The older one names the config `.json`; the OCI // one names it `blobs/sha256/`. They carry the same sixty-four characters, and a // reader that understood only one would work until somebody upgraded their runtime. for _, config := range []string{digest + ".json", "blobs/sha256/" + digest} { saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) - id, err := ID(saved) + id, err := ArchiveID(saved) if err != nil { t.Fatalf("config %q: %v", config, err) } @@ -62,7 +67,10 @@ func TestTheImageIdIsReadFromTheSavedFile(t *testing.T) { } } -func TestTheSavedTagsAreReadForAPersonToRecognise(t *testing.T) { +// The tag is read, and it is not decoration: it is the name the installer asks a runtime by, +// because it is the one thing that survives `docker save` and `docker load` unchanged. The id +// does not. +func TestTheSavedTagsAreRead(t *testing.T) { saved := savedImage(t, map[string]string{ "manifest.json": manifest(t, strings.Repeat("b", 64)+".json", "mesh-control:v1"), }) @@ -74,18 +82,18 @@ func TestTheSavedTagsAreReadForAPersonToRecognise(t *testing.T) { // A tar that is not a saved image is refused with what is wrong, not with a nil id. // -// The installer names the control plane by this id in the bundle it writes. An id it could not -// read, treated as empty, would produce a bundle naming nothing — refused by the host two steps -// later, with a message about a declaration rather than about what somebody embedded. +// Nothing here reaches a bundle any more, but this is still the earliest moment somebody can be +// told they embedded the wrong file — and the alternative is finding out at the load, from a +// container runtime, in a sentence about a tar rather than about what was built. func TestSomethingThatIsNotASavedImageIsRefused(t *testing.T) { notAnImage := savedImage(t, map[string]string{"hello": "world"}) - if _, err := ID(notAnImage); err == nil { + if _, err := ArchiveID(notAnImage); err == nil { t.Error("a tar with no manifest.json was accepted as a saved image") } else if !strings.Contains(err.Error(), "docker save") { t.Errorf("the refusal does not say what to embed instead: %v", err) } - if _, err := ID([]byte("this is not a tar at all")); err == nil { + if _, err := ArchiveID([]byte("this is not a tar at all")); err == nil { t.Error("bytes that are not a tar were accepted") } } @@ -101,7 +109,7 @@ func TestATarHoldingSeveralImagesIsRefused(t *testing.T) { t.Fatal(err) } saved := savedImage(t, map[string]string{"manifest.json": string(entries)}) - if _, err := ID(saved); err == nil { + if _, err := ArchiveID(saved); err == nil { t.Error("a tar holding two images was accepted") } } @@ -112,7 +120,7 @@ func TestATarHoldingSeveralImagesIsRefused(t *testing.T) { func TestAConfigThatIsNotADigestIsRefused(t *testing.T) { for _, config := range []string{"config.json", "abc.json", "blobs/sha256/" + strings.Repeat("a", 63)} { saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) - if _, err := ID(saved); err == nil { + if _, err := ArchiveID(saved); err == nil { t.Errorf("config %q was accepted and is not a digest", config) } } -- 2.54.0 From 7e3481f025d5533e0a30a7cdf8afbbc0a240c494 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 11:38:15 +0200 Subject: [PATCH 8/9] bootstrap: the slot the installer fills is not a registry to reach for The first real run of mesh-bootstrap stopped in preflight, dialling 192.0.2.250:5000 for ninety seconds on a machine whose network was fine. That address is the registry the lab used to raise; the substrate template still names the control plane by it, and step 3 replaces that reference with the id of the image this installer carries. Nothing ever pulls it. So preflight excludes the control plane's resource by identity, rather than by the happy accident of the template filling its slot with something that needs no registry. Every other container's registry is still dialled, because those are somebody else's images at somebody else's registry and a machine that cannot reach one fails inside a pull, which says the wrong thing. Also: `make bootstrap` takes BOOTSTRAP_OUT. The lab now builds the installer from source before every raise, into a path it chooses, and a caller that could not say where the output goes would have to copy it afterwards. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- Makefile | 12 ++++++++++-- internal/bootstrap/preflight.go | 16 +++++++++++++++- internal/bootstrap/preflight_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1c5f2c2..f65db19 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,14 @@ host: # hold, and the tag is the only name that survives the transfer. Saving by id produces an archive # with no tags at all, which the installer refuses; caught here instead, in front of the person who # can fix it. +# +# BOOTSTRAP_OUT is where the binary is written, and it exists because something other than a person +# now builds this: the lab rebuilds every artifact it runs from source before a raise, into paths it +# chose (mesh-lab's src/rebuild.ts, and novox/hq 04-ISSUES/005 for why it does that at all). A +# caller that could not say where the output goes would have to copy it afterwards, which is one +# more step to forget. +BOOTSTRAP_OUT ?= mesh-bootstrap + bootstrap: @test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; } @case "$(IMAGE)" in sha256:*) echo "IMAGE=$(IMAGE) is an image id. The installer identifies the carried image by its tag, because an id is the digest of a configuration that a runtime rewrites as it loads. Pass a name:tag"; exit 1;; esac @@ -82,11 +90,11 @@ bootstrap: @test -n "$$(docker image inspect --format '{{len .RepoTags}}' "$(IMAGE)" | grep -v '^0$$')" || { echo "$(IMAGE) has no repository tag, so the saved archive would carry no name the installer can ask a runtime about. Tag it first: docker tag $(IMAGE) mesh-control:"; exit 1; } @cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder @docker save --output internal/image/control-plane.tar "$(IMAGE)" - @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o mesh-bootstrap ./cmd/mesh-bootstrap; \ + @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o "$(BOOTSTRAP_OUT)" ./cmd/mesh-bootstrap; \ status=$$?; \ mv internal/image/control-plane.tar.placeholder internal/image/control-plane.tar; \ exit $$status - @echo "built mesh-bootstrap carrying $(IMAGE)" + @echo "built $(BOOTSTRAP_OUT) carrying $(IMAGE)" clean: rm -f mesh-host mesh-bootstrap diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index e13e276..4c0bb21 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -104,6 +104,15 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte // reason. Everything else is somebody else's image at somebody else's registry, and a machine // that cannot reach it fails inside a pull, which reports a network error where a person // reads a missing image. + // + // "The mesh's own image is skipped" used to mean "skipped if the template happened to name it + // in a way that needs no registry", and that is not the same sentence. A template names the + // control plane by SOMETHING — the reference is a slot, and step 3 replaces whatever is in it + // with the id of the image this installer carries. Whatever the slot held is therefore never + // pulled, never fetched, and never reached; requiring it to be reachable refuses a correct + // install because of a string that is about to be thrown away. Found on the first real run: the + // lab's template still carried `192.0.2.250:5000/mesh-control@…`, the address of a registry that + // no longer exists, and preflight timed out dialling it. for _, host := range registriesIn(parsed) { dialing, cancel := context.WithTimeout(ctx, o.Timeout) err := d.Dial(dialing, host) @@ -171,12 +180,17 @@ func containerRuntimeDetector(run Runner) profile.Detector { // registriesIn is every host the bundle's images would be fetched from, without duplicates and in // the order they appear. +// +// The control plane's own resource is excluded by identity rather than by the shape of what it +// names. Its image reference is a slot the installer overwrites with the id of the image it +// carries, so no registry ever serves it — and a template that filled that slot with a registry +// this machine cannot reach is not a machine with a network problem. func registriesIn(d *declaration.Declaration) []string { var hosts []string seen := map[string]bool{} for _, r := range d.Resources { container, ok := r.(*declaration.Container) - if !ok { + if !ok || container.Identity() == ControlPlaneID { continue } host, served := registryOf(container.Image) diff --git a/internal/bootstrap/preflight_test.go b/internal/bootstrap/preflight_test.go index 627dce0..1664da1 100644 --- a/internal/bootstrap/preflight_test.go +++ b/internal/bootstrap/preflight_test.go @@ -100,6 +100,31 @@ func TestOnlyTheRegistriesTheBundleNamesAreAskedAbout(t *testing.T) { } } +// And the control plane's slot is excluded whatever is in it. +// +// Found on the first real run of this installer. A template names the control plane by SOMETHING +// and step 3 replaces it with the id of the carried image, so whatever was there is never pulled — +// but preflight was reading that slot like any other and dialling it. The lab's template still +// carried the address of a registry the lab no longer raises, so a correct install timed out in +// preflight against a machine with a perfectly good network. +func TestTheControlPlanesOwnRegistryIsNeverAskedAbout(t *testing.T) { + parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"}, + {"id":"control-plane","type":"container","name":"mesh-control","image":"192.0.2.250:5000/mesh-control@sha256:` + + strings.Repeat("8", 64) + `"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + got := registriesIn(parsed) + if len(got) != 1 || got[0] != DefaultRegistry { + t.Fatalf("asked about %v; the control plane's own reference is about to be replaced and "+ + "must not be reached for", got) + } +} + func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) { // The container runtime's own rule: the part before the first slash is a registry host if it // has a dot, a port, or is localhost. Getting this wrong means dialling a hostname that is -- 2.54.0 From 26ff447aa3d6776a5453c072544f8603c837b54c Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 12:11:57 +0200 Subject: [PATCH 9/9] bootstrap: the mesh hearing from a machine is not an agent running on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enrolling IS the machine speaking to the mesh, so straight after it the mesh has always heard from this node — and the step took that as proof an agent was running and skipped starting one. The cost is silent and total. Everything after is the control plane being told things, and nothing it is told reaches a machine with no agent to collect it: the registry push at step 7 was accepted, the module recorded, and no container ever created. It surfaced three minutes later as 'the registry is not there at all', one step from its cause and looking nothing like it. Both halves are asked now. A process may be wedged and collect nothing, which is why the mesh is asked at all; and the mesh may have heard once from a machine running nothing, which is why the machine is asked too. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/bootstrap/enrol.go | 50 ++++++++++++++++++++++++++++---- internal/bootstrap/enrol_test.go | 40 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index 2387ec3..facbda5 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -154,14 +154,32 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane, say func(string)) (string, error) { - // Asked first, and of the mesh rather than of the process table. What matters is not that a - // process exists but that the mesh has heard from it, and only one of those two is the thing - // every later step depends on. + // **Asked of both, because either one alone lies.** + // + // A process in the table may be wedged and never collect anything, which is why this asked the + // mesh instead. But the mesh having heard from a node proves only that something spoke to it + // *once*, and `mesh-host enrol` — run moments earlier, by this very step — is itself that + // something. So straight after enrolling, the mesh has always heard from this machine and no + // agent is running; skipping the start on that signal alone is exactly wrong. + // + // What it costs is silent and total. Every later step is the control plane being *told* + // things, and nothing it is told reaches a machine with no agent to collect it: the push at + // step 7 is accepted, the mesh records the module, and no container is ever created. It + // surfaces three minutes later as "the registry is not there at all" — one step from its + // cause, looking nothing like it. if heard, err := heardFrom(ctx, control, o.Node); err != nil { return "", err } else if heard { - say(" host running the mesh has heard from " + o.Node) - return "already running", nil + running, err := agentIsRunning(ctx, o, sys, control) + if err != nil { + return "", err + } + if running { + say(" host running the mesh has heard from " + o.Node) + return "already running", nil + } + say(" host running the mesh has heard from " + o.Node + ", and no agent is running " + + "here — enrolling speaks once, which is not the same thing") } how := "" @@ -242,6 +260,28 @@ func runTheHost(ctx context.Context, o Options, sys system.System, control contr } } +// agentIsRunning is whether a host agent is on this machine now — the other half of the question +// the mesh cannot answer. +// +// Deliberately not an error when there is nothing to find: "no unit here" and "not running" are +// both simply *not running* to this caller, and the branch that starts one says far more about a +// missing unit than this could, naming what to install. +func agentIsRunning(ctx context.Context, o Options, sys system.System, control controlPlane) (bool, error) { + if o.HostInBackground { + // No unit to ask, so the process table is all there is. The exit status is the whole + // answer; anything it printed is not this function's business. + if _, err := control.run(ctx, "pgrep", "-f", o.Host+" run"); err != nil { + return false, nil + } + return true, nil + } + state, err := sys.ServiceState(ctx, control.run, o.HostService) + if err != nil { + return false, nil + } + return state == "running", nil +} + // heardFrom asks the mesh whether this node has spoken to it. func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) { listing, err := control.tell(ctx, "node", "list") diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go index 7561d56..cad2fa9 100644 --- a/internal/bootstrap/enrol_test.go +++ b/internal/bootstrap/enrol_test.go @@ -59,12 +59,17 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { switch { case strings.Contains(joined, "node list"): return "anchor here 01J0\n", nil + // An agent is running here too. Both halves are needed: enrolling makes the mesh hear from + // a machine once, so the first answer alone also describes a machine with no agent at all. + case name == "pgrep": + return "4242\n", nil } return "", fmt.Errorf("unexpected: %s %v", name, args) }} out, err := Enrol(context.Background(), Options{ Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, func(string) {}) if err != nil { @@ -82,6 +87,41 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { } } +// The mesh having heard from a machine is not the same as an agent running on it, and enrolling is +// itself the thing that makes the mesh hear. Taken as proof of life it ends the step believing an +// agent it never started, and everything after is the control plane being told things that never +// reach the machine: the next push is accepted, recorded, and applied by nobody. +func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) { + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + // Exactly what enrolling leaves behind, with nothing running. + return "anchor here 01J0\n", nil + case name == "pgrep": + return "", fmt.Errorf("exit status 1") + case name == "sh": + return "", nil + } + return "", fmt.Errorf("unexpected: %s %v", name, args) + }} + + out, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Agent == "already running" { + t.Fatal("a machine with no agent was reported as already running it") + } + if !runtime.ran("nohup") { + t.Errorf("no agent was started on a machine that has none: %v", runtime.commands) + } +} + // A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling // replaces the identity the mesh recorded, which is a deliberate act and not something an // installer does on its own. -- 2.54.0