diff --git a/.gitignore b/.gitignore index 235d26c..d9cc9a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /mesh-host +/mesh-bootstrap /dist/ +# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an +# interrupted release build cannot commit a twenty-megabyte tar by accident. +/internal/image/control-plane.tar.placeholder diff --git a/Makefile b/Makefile index ebe96e7..d5bf716 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LDFLAGS := -s -w -X main.builtFor=$(SYSTEM) -X main.version=$(VERSION) # a second file to arrive with it is not "copy it and run it". BUNDLE ?= -.PHONY: check test vet fmt build clean host +.PHONY: check test vet fmt build clean host hosts bootstrap packaging-test check: fmt vet test packaging-test build @@ -57,5 +57,27 @@ host: exit $$status @echo "built for $(SYSTEM) carrying $(BUNDLE)" +# The installer, carrying the control plane's image: +# make bootstrap IMAGE=mesh-control:v1.2.3 +# +# The image is BUILT ELSEWHERE and handed over — mesh-control's own `make image` — and embedded +# here at release time. Not built on the machine being bootstrapped, and not fetched: the forge +# that holds mesh-control's source runs on the mesh, so a bootstrap that had to fetch or build the +# control plane would need a mesh in order to raise one. Carrying it breaks that cycle, the same +# way carrying the bundle breaks the "copy it onto a machine and run it" one (novox/hq ADR 0005). +# +# The saved image occupies the embed slot for the length of one build and the placeholder goes +# back, exactly as `host:` does with the bundle. Nothing large is ever committed. +bootstrap: + @test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; } + @docker image inspect "$(IMAGE)" >/dev/null 2>&1 || { echo "this machine does not hold $(IMAGE) — build it in mesh-control with 'make image'"; exit 1; } + @cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder + @docker save --output internal/image/control-plane.tar "$(IMAGE)" + @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o mesh-bootstrap ./cmd/mesh-bootstrap; \ + status=$$?; \ + mv internal/image/control-plane.tar.placeholder internal/image/control-plane.tar; \ + exit $$status + @echo "built mesh-bootstrap carrying $(IMAGE)" + clean: - rm -f mesh-host + rm -f mesh-host mesh-bootstrap diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go new file mode 100644 index 0000000..894462c --- /dev/null +++ b/cmd/mesh-bootstrap/main.go @@ -0,0 +1,192 @@ +// Command mesh-bootstrap brings a mesh into existence on a bare machine. +// +// Tier 0, beside `mesh-host` and not inside it. Bootstrapping is done by hand and it changes a +// machine, which is what tier 0 is (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) — but +// `mesh-host` states of itself that it connects to nothing and listens on nothing and that what it +// applies comes from a file, and that is the whole reason an always-running root daemon can be +// audited by reading one page. An installer that loads images and interrogates a control plane +// cannot be folded into it without making that sentence false. Same tier, same repository, +// different program. +// +// What it does not do is enrol this machine, register modules or assign them. It stops at a +// running substrate with a control plane that answers, which is a mesh of one node. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net" + "os" + "os/signal" + "syscall" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// version is stamped at build time. Unset in a development build, and said so rather than +// defaulted to something that looks like a release. +var version = "development build" + +const ( + defaultTemplate = "substrate.lock" + defaultOut = "/var/lib/mesh-host/substrate.lock" +) + +const usage = `mesh-bootstrap — make a bare machine into a mesh + + bootstrap preflight, load, bundle, apply, verify (the default) + version + + --bundle the substrate template to build this machine's bundle from + (default ` + defaultTemplate + `) + --out where the produced bundle is written, for a person to read + (default ` + defaultOut + `) + --state where this node records what it has applied + (default ` + store.DefaultPath + `) + --system which operating system this is; by default it is asked + --timeout how long any single probe may take (default 30s) + --wait how long a thing that is merely starting is given (default 3m) + --dry-run everything that does not change the machine + --json machine-readable output + +It carries the control plane's image and applies a substrate. Every step is idempotent: +run it again after fixing whatever it named, and the steps that already succeeded say so. +` + +func main() { + // Ctrl-C must stop the installer, not be swallowed by whatever it is waiting for — and it + // waits on pulls, on a runtime starting, and on a control plane opening its stores. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + command, opts, jsonOut, err := parseArgs(os.Args[1:]) + if err == nil { + err = run(ctx, command, opts, jsonOut) + } + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-bootstrap: %v\n", err) + os.Exit(1) + } +} + +// parseArgs takes an optional subcommand first, then its flags. +// +// Parsed in a loop for the reason `mesh-host` records: the standard library stops at the FIRST +// non-flag argument, so a flag sitting after one is silently dropped and the command exits zero +// having ignored what it was asked. That fault has been paid for twice in this repository and is +// not being paid for a third time. +func parseArgs(args []string) (string, bootstrap.Options, bool, error) { + opts := bootstrap.Options{ + Template: defaultTemplate, + Out: defaultOut, + State: store.DefaultPath, + // Longer than the host's 10s: these probes reach a container runtime that may be busy + // pulling, and a probe that times out on a working machine is a false refusal. + Timeout: 30 * time.Second, + // A socket-activated runtime queued behind the network, and a control plane running its + // first `initdb`-shaped wait, are both minutes rather than seconds. + Wait: 3 * time.Minute, + } + var jsonOut bool + + command := "bootstrap" + if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' { + command = args[0] + args = args[1:] + } + + set := newFlagSet(&opts, &jsonOut) + var positionals []string + rest := args + for { + if err := set.Parse(rest); err != nil { + return "", opts, false, err + } + rest = set.Args() + if len(rest) == 0 { + break + } + positionals = append(positionals, rest[0]) + rest = rest[1:] + } + + // Refused rather than ignored: a mistyped argument that changes nothing and reports success is + // worse than an error, and this program's whole job is to change a machine. + if len(positionals) > 0 { + return "", opts, false, fmt.Errorf( + "unexpected argument %q — try `mesh-bootstrap help`", positionals[0]) + } + return command, opts, jsonOut, nil +} + +func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { + set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError) + set.SetOutput(os.Stderr) + set.Usage = func() { fmt.Fprint(os.Stderr, usage) } + set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from") + set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written") + set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied") + set.StringVar(&opts.System, "system", opts.System, "which operating system this is") + set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take") + set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given") + set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine") + set.BoolVar(jsonOut, "json", false, "machine-readable output") + return set +} + +func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error { + switch command { + case "bootstrap": + say := func(line string) { + if !jsonOut { + fmt.Println(line) + } + } + result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{ + Run: apply.ExecRunner, + Dial: dial, + }, say) + + // Printed whichever way it went. What the installer got through before it stopped is on + // the machine either way, and a report that only exists on success describes a machine + // nobody has (novox/hq ADR 0018). + if jsonOut { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if encodeErr := encoder.Encode(result); encodeErr != nil && err == nil { + return encodeErr + } + } + return err + + case "version": + fmt.Println(version) + return nil + + case "help", "-h", "--help": + fmt.Fprint(os.Stderr, usage) + return nil + + default: + return fmt.Errorf("unknown command %q — try `mesh-bootstrap help`", command) + } +} + +// dial answers whether a TCP address responds. +// +// A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a +// connection to exactly this address. A machine whose DNS resolves and whose route is missing +// passes a lookup and fails the thing that matters. +func dial(ctx context.Context, address string) error { + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return err + } + return conn.Close() +} diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go new file mode 100644 index 0000000..6731428 --- /dev/null +++ b/cmd/mesh-bootstrap/main_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "flag" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// Argument handling gets tests for the reason `mesh-host` records: the standard library stops +// parsing at the first non-flag argument, so a flag sitting after one is silently dropped and the +// command exits zero having ignored what it was asked. Here that would mean `--dry-run` ignored on +// a program whose whole job is to change a machine. + +func TestBootstrapIsWhatItDoesWithNoCommand(t *testing.T) { + // Running the installer with nothing but flags must install, not print usage: the command is + // the reason the binary exists, and making somebody type its name twice buys nothing. + command, opts, _, err := parseArgs([]string{"--dry-run"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if command != "bootstrap" { + t.Errorf("command = %q, want bootstrap", command) + } + if !opts.DryRun { + t.Error("--dry-run before any subcommand was ignored") + } +} + +func TestFlagsAreReadWhereverTheySit(t *testing.T) { + for _, args := range [][]string{ + {"bootstrap", "--dry-run", "--bundle", "s.lock", "--json"}, + {"bootstrap", "--json", "--bundle=s.lock", "--dry-run"}, + {"--bundle", "s.lock", "--dry-run", "--json"}, + } { + _, opts, jsonOut, err := parseArgs(args) + if err != nil { + t.Errorf("%v: unexpected error: %v", args, err) + continue + } + if !opts.DryRun || !jsonOut || opts.Template != "s.lock" { + t.Errorf("%v parsed as dry-run=%v json=%v bundle=%q", + args, opts.DryRun, jsonOut, opts.Template) + } + } +} + +func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) { + // Asymmetric cost: an error is a moment's annoyance, and a silently dropped --dry-run is a + // machine changed by somebody who asked for it not to be. + if _, _, _, err := parseArgs([]string{"bootstrap", "--dry-runn"}); err == nil { + t.Fatal("a mistyped flag was accepted") + } +} + +func TestAnUnexpectedArgumentIsRefused(t *testing.T) { + if _, _, _, err := parseArgs([]string{"bootstrap", "substrate.lock"}); err == nil { + t.Fatal("a stray argument was ignored rather than refused — the bundle is --bundle") + } +} + +func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) { + // The usage text is a promise. A default that drifts from what is printed is a small lie that + // costs somebody an afternoon in front of a machine that will not come up. + _, opts, jsonOut, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Template != defaultTemplate { + t.Errorf("default bundle is %q; the usage text says %q", opts.Template, defaultTemplate) + } + if opts.Out != defaultOut { + t.Errorf("default out is %q; the usage text says %q", opts.Out, defaultOut) + } + if opts.State != store.DefaultPath { + t.Errorf("default state is %q; the host's own default is %q", opts.State, store.DefaultPath) + } + if opts.Timeout != 30*time.Second { + t.Errorf("default timeout is %s; the usage text says 30s", opts.Timeout) + } + if opts.Wait != 3*time.Minute { + t.Errorf("default wait is %s; the usage text says 3m", opts.Wait) + } + if opts.DryRun || jsonOut || opts.System != "" { + t.Error("something is on by default that the usage text describes as a flag") + } +} + +// Every flag the usage text promises must exist, and every flag that exists must be in the usage +// text. The two drifting apart is how a program acquires a feature nobody can find and a +// documented option that does nothing. +func TestTheUsageTextAndTheFlagsAgree(t *testing.T) { + var opts bootstrap.Options + var jsonOut bool + set := newFlagSet(&opts, &jsonOut) + + declared := map[string]bool{} + set.VisitAll(func(f *flag.Flag) { declared[f.Name] = true }) + + for name := range declared { + if !strings.Contains(usage, "--"+name) { + t.Errorf("--%s exists and the usage text does not mention it", name) + } + } + for _, promised := range []string{"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json"} { + if !declared[promised] { + t.Errorf("the usage text promises --%s and no such flag exists", promised) + } + } +} diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go new file mode 100644 index 0000000..e792059 --- /dev/null +++ b/internal/bootstrap/apply.go @@ -0,0 +1,134 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Runner is the same runner every applier in this repository takes. +type Runner = apply.Runner + +// ApplyBundle raises the substrate, through the host's own apply. +// +// **This calls `internal/apply` rather than running the `mesh-host` binary**, and that is worth +// stating because shelling out would have been easier. The installer and the host must apply a +// declaration identically — same removal pass, same read-backs, same refusal model, same record of +// what this machine now owns — and two code paths that must behave the same are two code paths +// that will not. The `mesh-host` binary is also not guaranteed to be on a machine this program is +// raising, which would make the installer depend on the thing it installs. +// +// It applies under `store.OriginCarried`, which is the same origin `mesh-host reconcile` uses and +// is not a detail: what the substrate raised must be invisible to the removal pass of a +// declaration that later arrives from the control plane, or the first thing the mesh tells this +// node would tear down the mesh (novox/hq 04-ISSUES/010). +// +// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version, +// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is +// not one. +func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration, + source string, run Runner, say func(string)) (apply.Report, error) { + + // Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host` + // does: finding out half way through is the half-configured machine tier 0 exists to prevent. + if err := system.Check(sys, d); err != nil { + return apply.Report{}, err + } + + known, err := store.Load(o.State) + if err != nil { + return apply.Report{}, err + } + + report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + + // Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a + // failure is on the machine either way, and a host that did not record it would believe it + // owns less than it does and leave that behind for ever. + if saveErr := store.Save(o.State, updated); saveErr != nil { + if applyErr != nil { + return report, fmt.Errorf("%w\n\nand this node's state could not be saved: %v", + applyErr, saveErr) + } + return report, saveErr + } + if applyErr != nil { + return report, fmt.Errorf("%w\n\nThe machine is in whatever state that left it. Fix what "+ + "is named above and run this again — every step is idempotent, and the ones that "+ + "already succeeded will say so", applyErr) + } + return report, nil +} + +// refuseSealed is what happens when a bundle contains a file the mesh sealed to this node. +// +// It cannot happen and it is refused with a sentence rather than a nil dereference. A sealing key +// is generated at enrolment (`internal/identity`), and enrolment is something that happens on a +// mesh — which is the thing this program is raising. A substrate bundle carrying a sealed file +// would be a bundle written for a node that has already joined. +func refuseSealed(string) ([]byte, error) { + return nil, errors.New( + "this bundle contains a file sealed to a node's key, and a machine that has not enrolled " + + "has no such key. A substrate is applied before any mesh exists, so it can carry no " + + "secret the mesh sealed") +} + +// WorkOutSystem decides which half of the host applies things on this machine, and proves it. +// +// `mesh-host` pins this at link time because it is built for one operating system and refuses to +// touch a machine without knowing which (novox/hq ADR 0005). An installer run by hand has no +// link-time to pin it at, so it asks — but it does not guess: every system already knows how to +// prove it is the one it claims to be, by asking its package database about a package that is +// certainly there. Exactly one may answer. +// +// A machine where none answers is refused with what each of them said, because "unsupported +// system" is a sentence nobody can act on and "pacman does not answer here" is. +func WorkOutSystem(ctx context.Context, run Runner, named string) (system.System, error) { + if strings.TrimSpace(named) != "" { + chosen, err := system.For(named) + if err != nil { + return nil, err + } + if err := chosen.Confirm(ctx, run); err != nil { + return nil, fmt.Errorf("--system %s was given, and this machine says otherwise: %w", + named, err) + } + return chosen, nil + } + + var answered []system.System + var refusals []string + for _, candidate := range system.All() { + if err := candidate.Confirm(ctx, run); err != nil { + refusals = append(refusals, fmt.Sprintf(" %s: %v", candidate.Name(), err)) + continue + } + answered = append(answered, candidate) + } + + switch len(answered) { + case 1: + return answered[0], nil + case 0: + return nil, fmt.Errorf( + "this machine is none of the systems this installer knows how to change, so nothing "+ + "was attempted:\n%s\nName one with --system if it is really one of them and its "+ + "package database is merely unwell", strings.Join(refusals, "\n")) + default: + var names []string + for _, s := range answered { + names = append(names, s.Name()) + } + return nil, fmt.Errorf( + "this machine answers as %s at once, and the installer must not choose between them: "+ + "package names and unit names differ, and picking wrong misconfigures the machine "+ + "quietly. Say which with --system", strings.Join(names, " and ")) + } +} diff --git a/internal/bootstrap/apply_test.go b/internal/bootstrap/apply_test.go new file mode 100644 index 0000000..7a3c8ea --- /dev/null +++ b/internal/bootstrap/apply_test.go @@ -0,0 +1,91 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" +) + +// `mesh-host` is built for one operating system and pins it at link time. An installer run by hand +// has no link time, so it asks — and it does not guess: each system already knows how to prove it +// is the one it claims to be, by asking its package database about a package that is certainly +// there. Getting this wrong installs with the wrong package manager and the wrong unit names. + +func TestTheMachineIsAskedWhichSystemItIs(t *testing.T) { + // Only pacman answers, so this is the arch host and nothing had to be told so. + onlyPacman := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "pacman" { + return "pacman 7.0.0-1\n", nil + } + return "", errors.New("command not found") + } + + chosen, err := WorkOutSystem(context.Background(), onlyPacman, "") + if err != nil { + t.Fatal(err) + } + if chosen.Name() != "arch" { + t.Errorf("this machine was worked out to be %q", chosen.Name()) + } +} + +// A machine that is none of them is refused with what each of them said. "Unsupported system" is +// a sentence nobody can act on; "pacman does not answer here" is. +func TestAMachineThatIsNoneOfThemIsRefusedWithWhatEachSaid(t *testing.T) { + nothing := func(context.Context, string, ...string) (string, error) { + return "", errors.New("command not found") + } + + _, err := WorkOutSystem(context.Background(), nothing, "") + if err == nil { + t.Fatal("a machine that answers as no known system was accepted") + } + for _, wanted := range []string{"arch:", "alpine:", "--system"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a machine that was TOLD what it is still has to prove it. Installing the arch half of the +// host on Alpine must say so once, at the start, rather than failing later inside pacman. +func TestASystemThatWasNamedIsStillProved(t *testing.T) { + onlyApk := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "apk" { + return "apk-tools-2.14.0\n", nil + } + return "", errors.New("command not found") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "arch"); err == nil { + t.Fatal("--system arch was believed on a machine where pacman does not answer") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "alpine"); err != nil { + t.Errorf("--system alpine was refused on a machine where apk answers: %v", err) + } +} + +func TestASystemNobodyHasBuiltIsRefusedByName(t *testing.T) { + anything := func(context.Context, string, ...string) (string, error) { return "", nil } + _, err := WorkOutSystem(context.Background(), anything, "debian") + if err == nil { + t.Fatal("--system debian was accepted, and no debian host is built") + } + if !strings.Contains(err.Error(), "arch") { + t.Errorf("the refusal does not say which systems exist: %v", err) + } +} + +// A substrate is applied before any mesh exists, so it can carry no secret the mesh sealed — there +// is no key to open one with. Refused with a sentence rather than a nil dereference. +func TestASealedFileInASubstrateIsRefusedWithAReason(t *testing.T) { + _, err := refuseSealed("anything") + if err == nil { + t.Fatal("a sealed file in a substrate bundle was accepted") + } + if !strings.Contains(err.Error(), "has not enrolled") { + t.Errorf("the refusal does not say why there is no key: %v", err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 0000000..428aefc --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,260 @@ +// Package bootstrap brings a mesh into existence on a bare machine. +// +// **Why this is a program at all.** Until now the only complete written-down copy of the +// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is +// why every gap in it kept being found late and by accident: an install procedure that lives as a +// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is +// that procedure, made into the thing it always was. +// +// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by +// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is +// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and +// listens on nothing and that what it applies comes from a file — a property that is what makes an +// always-running root daemon auditable, and that must stay literally true. This program pulls +// images and asks a running control plane questions. Same tier, same repository, different binary. +// +// **No registry is required for the mesh's own image, and no source either.** The control plane +// exists in no registry by design, and the forge that holds its source runs on the mesh — so a +// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer +// carries the image (`internal/image`) and names it by its image id: the sha256 of its own +// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq +// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream +// `name@sha256:` references and are pulled from the internet like anything else. +package bootstrap + +import ( + "context" + "fmt" + "time" +) + +// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence +// nobody can act on and this will be run over and over by somebody getting a machine working. +type Step string + +const ( + StepPreflight Step = "preflight" + StepLoad Step = "load" + StepBundle Step = "bundle" + StepApply Step = "apply" + StepVerify Step = "verify" +) + +// Steps in the order they happen, so a failure can say "step 2 of 5". +var Steps = []Step{StepPreflight, StepLoad, StepBundle, StepApply, StepVerify} + +// Error is a failure, named by the step it happened in. +type Error struct { + Step Step + Err error +} + +func (e *Error) Error() string { + at := 0 + for i, s := range Steps { + if s == e.Step { + at = i + 1 + } + } + return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err) +} + +func (e *Error) Unwrap() error { return e.Err } + +func failed(step Step, err error) error { + if err == nil { + return nil + } + return &Error{Step: step, Err: err} +} + +// Options are the things that differ between machines. +type Options struct { + // Template is the substrate bundle this machine's own bundle is made from. + Template string + // Out is where the produced bundle is written, so a person can read what was applied. + Out string + // State is where the host records what it has applied here — the same file `mesh-host` reads, + // because what this raises the host must afterwards own. + State string + // System is which half of the host applies things. Empty means ask the machine. + System string + // DryRun does everything that does not change the machine. + DryRun bool + // Timeout bounds any single probe. + Timeout time.Duration + // Wait is how long something that is merely starting is given: a socket-activated container + // runtime, a control plane opening its stores. + Wait time.Duration +} + +// Deps are the ways this program reaches outside itself. Injected so the whole of it can be +// tested without a container runtime, a network, or a machine to break — the same reason +// `internal/apply` takes a Runner (novox/hq ADR 0017). +type Deps struct { + // Run executes a command. apply.ExecRunner in production. + Run Runner + // Dial reports whether a TCP address answers, for "can this machine reach the registries the + // bundle names". + Dial func(ctx context.Context, address string) error +} + +// Result is what the bootstrap did, in the shape `--json` prints. +type Result struct { + System string `json:"system"` + DryRun bool `json:"dry-run,omitempty"` + + // Image is the control plane's image id — what the produced bundle names it by. + Image string `json:"image,omitempty"` + // ImageTags is what that image was called when it was saved. Decoration, for a person. + ImageTags []string `json:"image-tags,omitempty"` + // ImageHeld is true when the machine already held it and nothing was loaded. + ImageHeld bool `json:"image-already-held,omitempty"` + + // Bundle is where the produced bundle was written, and what was done to produce it. + Bundle string `json:"bundle,omitempty"` + BundleWas string `json:"bundle-replaced,omitempty"` + BundlePlaces int `json:"bundle-places,omitempty"` + BundleWrote bool `json:"bundle-written,omitempty"` + + // Applied is how many resources the apply reported on, and whether any of them moved. + Applied int `json:"applied,omitempty"` + Changed bool `json:"changed,omitempty"` + + // Running is the substrate's containers, confirmed up. + Running []string `json:"running,omitempty"` + // Answered is what the control plane said back — not merely that it is up. + Answered string `json:"control-plane,omitempty"` + + // Stopped names why a dry run went no further. Empty on a real run. + Stopped string `json:"stopped,omitempty"` +} + +// Run performs the bootstrap, saying what it is doing as it goes. +// +// Every step is idempotent, and every step says whether it found something or changed it. That is +// not politeness: this program is run repeatedly while somebody gets a machine working, and a step +// that cannot tell "already done" from "just done" makes the second run indistinguishable from the +// first — which is how a person stops believing any of it. +// +// It does not retry. A pull that failed for a reason that goes away by itself is real, and the +// answer to it is to run this again: re-running is the retry, and it is one a person chooses after +// reading which step failed and why. +// +// **What this does NOT do: enrolment, the module catalogue, and assignment.** It stops at a running +// substrate with a control plane that replies — a mesh of one node with nothing joined to it. See +// the marker at the end. +func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) { + if say == nil { + say = func(string) {} + } + result := Result{DryRun: o.DryRun} + + // ---- 1. preflight ------------------------------------------------------------------- + say("preflight — what has to be true before anything is changed") + template, err := Preflight(ctx, o, d, say) + if err != nil { + return result, failed(StepPreflight, err) + } + + // Which half of the host applies things here. Asked of the machine and proved, because + // `mesh-host` pins this at link time and an installer run by hand has no link time. + sys, err := WorkOutSystem(ctx, d.Run, o.System) + if err != nil { + return result, failed(StepPreflight, err) + } + result.System = sys.Name() + say(" system " + sys.Name()) + + // ---- 2. load ------------------------------------------------------------------------ + say("load — the control plane's image, carried in this installer") + loaded, err := Load(ctx, d.Run, o.DryRun, say) + if err != nil { + return result, failed(StepLoad, err) + } + result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held + + // ---- 3. bundle ---------------------------------------------------------------------- + say("bundle — what this machine will be asked to be") + rewritten, err := Rewrite(template, loaded.ID) + if err != nil { + return result, failed(StepBundle, err) + } + result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out + if rewritten.Changed { + say(fmt.Sprintf(" control plane %s", rewritten.Now)) + say(fmt.Sprintf(" replacing %s, named in %d place(s)", + rewritten.Was, rewritten.Places)) + } else { + say(fmt.Sprintf(" control plane %s — the template already named it, nothing rewritten", + rewritten.Now)) + } + for _, kept := range rewritten.Kept { + say(" left alone " + kept) + } + if rewritten.BrokerAddress != "" { + // Said every time, and never changed. Every enrolment token this mesh issues will tell a + // joining node to dial this address, and a wrong one is silent until the second node fails + // to come back. The installer does not know this machine's address and will not invent it. + say(" nodes will dial " + rewritten.BrokerAddress + + " — check this is an address other machines can reach") + } + + if o.DryRun { + // Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it + // if wrong while changing nothing. The bundle has been produced and re-parsed in memory, + // which is everything that could be checked without touching the machine; what is left is + // loading, applying and asking the result questions, and none of those can be answered by + // not doing them. + say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources)) + result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + + "loaded or applied" + say("\n" + result.Stopped) + return result, nil + } + + if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil { + return result, failed(StepBundle, err) + } + result.BundleWrote = true + say(fmt.Sprintf(" wrote %s (%d resources) — read it, this is what is applied", + o.Out, rewritten.Resources)) + + // ---- 4. apply ----------------------------------------------------------------------- + say("apply — raising the substrate") + report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, o.Out, d.Run, say) + result.Applied, result.Changed = len(report.Outcomes), report.Changed() + if err != nil { + return result, failed(StepApply, err) + } + if report.Changed() { + say(fmt.Sprintf(" applied %d resource(s)", len(report.Outcomes))) + } else { + say(fmt.Sprintf(" already matches %d resource(s) checked, nothing moved", + len(report.Outcomes))) + } + + // ---- 5. verify ---------------------------------------------------------------------- + say("verify — the substrate is up, and the control plane replies") + verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say) + result.Running, result.Answered = verified.Running, verified.Answered + if err != nil { + return result, failed(StepVerify, err) + } + + say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + + // NEXT STAGE — NOT IMPLEMENTED HERE. + // + // What remains between "a mesh exists" and "a mesh does something": issuing this machine a + // token and enrolling it as its own first node, registering the module catalogue with the + // control plane, and assigning modules to nodes. All three are conversations with the control + // plane that has just been proved to reply, so they belong after this point and inside none of + // the steps above. + // + // Left out rather than half-written. Everything above changes a machine; all of that changes a + // mesh, and a program that did both would have two jobs and one name. + say("not done here: enrolment, the module catalogue, and assignment.") + + return result, nil +} diff --git a/internal/bootstrap/load.go b/internal/bootstrap/load.go new file mode 100644 index 0000000..a44db70 --- /dev/null +++ b/internal/bootstrap/load.go @@ -0,0 +1,122 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-host/internal/image" +) + +// Loaded is the control plane's image on this machine. +type Loaded struct { + // ID is what the bundle will name the image by: sha256 of its own configuration. + ID string + // Tags is what it was called when it was saved. For a person, never for the bundle. + Tags []string + // Held is true when the machine already had it and nothing was loaded. + Held bool +} + +// Load puts the carried control-plane image into this machine's container runtime. +// +// **Idempotent by asking first, which is possible because the id is a fact about the file.** The +// image id is read out of the saved tar (see `internal/image`.ID) before the runtime is asked +// anything, so this can ask "do you already hold exactly this image" — and on the second, third +// and tenth run of the installer the answer is yes and nothing is loaded. A load that scraped the +// id out of what `docker load` printed could only know that after loading, so it would load every +// time and report the same thing either way. +// +// It reads back (novox/hq ADR 0018). A load that reported success and left nothing there is a +// failure, not a convergence, and the apply would then meet a bundle naming an image the machine +// does not hold — which fails correctly but two steps too late. +func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loaded, error) { + saved, err := image.Saved() + if err != nil { + return Loaded{}, err + } + return loadImage(ctx, run, saved, dryRun, say) +} + +// loadImage is Load with the carried bytes handed in, so the whole path can be tested against a +// saved image a test builds rather than against whatever a particular build embedded. +func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say func(string)) (Loaded, error) { + id, err := image.ID(saved) + if err != nil { + return Loaded{}, err + } + loaded := Loaded{ID: id, Tags: image.Tags(saved)} + + if held, err := holdsImage(ctx, run, id); err != nil { + return loaded, err + } else if held { + loaded.Held = true + say(" already held " + id + " — nothing loaded") + return loaded, nil + } + + if dryRun { + say(fmt.Sprintf(" would load %s (%d bytes)", id, len(saved))) + return loaded, nil + } + + // Through a file rather than through stdin: the runner this repository shares runs a command + // and captures its output, and giving it a second mouth for one caller would change every + // applier's contract for the sake of one step (internal/apply's Runner). + tarball, err := os.CreateTemp("", "mesh-control-*.tar") + if err != nil { + return loaded, fmt.Errorf("nowhere to put the carried image while loading it: %w", err) + } + defer os.Remove(tarball.Name()) + + if _, err := tarball.Write(saved); err != nil { + tarball.Close() + return loaded, fmt.Errorf("cannot write the carried image to %s: %w", tarball.Name(), err) + } + if err := tarball.Close(); err != nil { + return loaded, fmt.Errorf("cannot finish writing %s: %w", tarball.Name(), err) + } + + if _, err := run(ctx, "docker", "load", "--input", tarball.Name()); err != nil { + return loaded, fmt.Errorf( + "the container runtime would not load the carried control-plane image: %w", err) + } + + // Read back. This is what makes "loaded" a fact rather than an intention. + held, err := holdsImage(ctx, run, id) + if err != nil { + return loaded, err + } + if !held { + return loaded, fmt.Errorf( + "the load reported success and this machine does not hold %s.\n"+ + "The bundle names the control plane by that id and nothing serves it, so the "+ + "apply would refuse. Check what `docker load` actually took", id) + } + say(" loaded " + id) + return loaded, nil +} + +// holdsImage asks the runtime whether this exact image is present. +// +// It asks for the id back rather than reading the exit code, because an image inspected by id and +// an image inspected by a tag that happens to point somewhere else are the same successful +// command. What is wanted is "this one", and the answer says which one. +func holdsImage(ctx context.Context, run Runner, id string) (bool, error) { + out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", id) + if err != nil { + // Absent is an answer, not a failure. Every other reason the runtime might refuse looks + // the same from here — which is why preflight proves the runtime answers before this runs, + // rather than this trying to tell the two apart from an exit code. + return false, nil + } + got := strings.TrimSpace(out) + if got != id { + return false, fmt.Errorf( + "asked for image %s, the runtime answered %q. An image id is the digest of the "+ + "image's own configuration, so these are two different images and the bundle "+ + "would name the wrong one", id, got) + } + return true, nil +} diff --git a/internal/bootstrap/load_test.go b/internal/bootstrap/load_test.go new file mode 100644 index 0000000..0442875 --- /dev/null +++ b/internal/bootstrap/load_test.go @@ -0,0 +1,211 @@ +package bootstrap + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/image" +) + +// Docker is never required here. What is being tested is which commands the installer issues and +// what it concludes from the answers, so the runtime is injected the way `internal/apply` injects +// its Runner (novox/hq ADR 0017). Behaviour against a real runtime is proved in the lab. + +// asked records every command, so a test can assert that something was NOT run — which is the +// whole of what idempotence means here. +type asked struct { + commands []string + answer func(name string, args []string) (string, error) +} + +func (a *asked) run(_ context.Context, name string, args ...string) (string, error) { + a.commands = append(a.commands, strings.TrimSpace(name+" "+strings.Join(args, " "))) + if a.answer == nil { + return "", errors.New("this test did not expect any command to be run") + } + return a.answer(name, args) +} + +func (a *asked) ran(fragment string) bool { + for _, command := range a.commands { + if strings.Contains(command, fragment) { + return true + } + } + return false +} + +func savedImageFixture(t *testing.T, digest string) []byte { + t.Helper() + entries, err := json.Marshal([]struct { + Config string + RepoTags []string + }{{Config: digest + ".json", RepoTags: []string{"mesh-control:test"}}}) + if err != nil { + t.Fatal(err) + } + + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + if err := writer.WriteHeader(&tar.Header{ + Name: "manifest.json", Mode: 0o644, Size: int64(len(entries)), + }); err != nil { + t.Fatal(err) + } + if _, err := writer.Write(entries); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +const fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333" + +// A machine that already holds the image is not loaded again, and says so. +// +// This is the idempotence the installer's usefulness rests on: it is run over and over while +// somebody gets a machine working, and a step that did its work again every time would be +// indistinguishable from one that had never run. +func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "sha256:" + fixtureDigest + "\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + + if !loaded.Held { + t.Error("the machine already held the image and the load did not say so") + } + if runtime.ran("docker load") { + t.Errorf("the image was loaded again although the machine held it: %v", runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "already held") { + t.Errorf("nothing was said about finding the image already there: %v", said) + } +} + +// The id comes out of the file, and the bundle is named by it. +// +// Not scraped from what `docker load` prints — that is a sentence for a person, which reads +// `Loaded image: name:tag` or `Loaded image ID: sha256:…` depending on how the image was saved. +// A program depending on which one a runtime chose would be depending on a runtime version. +func TestTheImageIdComesFromTheCarriedFileNotFromWhatTheRuntimeSays(t *testing.T) { + runtime := &asked{} + inspected := 0 + runtime.answer = func(_ string, args []string) (string, error) { + switch { + case len(args) > 1 && args[0] == "image" && args[1] == "inspect": + inspected++ + if inspected == 1 { + return "", errors.New("Error: No such image") + } + return "sha256:" + fixtureDigest + "\n", nil + case len(args) > 0 && args[0] == "load": + // Deliberately says something else entirely. The id must not come from here. + return "Loaded image: some-other-name:whatever\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + } + + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("the image id is %q, want sha256:%s", loaded.ID, fixtureDigest) + } + if loaded.Held { + t.Error("an image that had to be loaded was reported as already held") + } + if !runtime.ran("docker load") { + t.Errorf("the image was never loaded: %v", runtime.commands) + } +} + +// A load that reported success and left nothing there is a failure, not a convergence +// (novox/hq ADR 0018). Without the read-back it would surface later as the host refusing a bundle +// naming an image nothing serves — a true message about the wrong thing. +func TestALoadThatLeftNothingBehindIsAFailure(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "Loaded image: mesh-control:test\n", nil + }} + + _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}) + if err == nil { + t.Fatal("a load that left nothing on the machine was reported as success") + } + if !strings.Contains(err.Error(), fixtureDigest) { + t.Errorf("the failure does not say which image is missing: %v", err) + } +} + +// A dry run changes nothing, and still knows the id — because the id is a property of the carried +// file. That is what lets `--dry-run` produce and check the real bundle rather than a guess. +func TestADryRunLearnsTheIdAndLoadsNothing(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "", fmt.Errorf("a dry run ran %v", args) + }} + + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), true, func(string) {}) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("a dry run did not work out the image id: %q", loaded.ID) + } + if runtime.ran("docker load") { + t.Errorf("a dry run loaded an image: %v", runtime.commands) + } +} + +// An installer built from a plain checkout carries no image, and says which build step is missing. +// Discovered here, before a machine is touched, rather than after a bundle has been written. +func TestAnInstallerCarryingNoImageSaysSoRatherThanRaisingHalfAMesh(t *testing.T) { + if !image.IsEmpty() { + t.Skip("this checkout has a saved image embedded") + } + _, err := Load(context.Background(), (&asked{}).run, false, func(string) {}) + if !errors.Is(err, image.ErrEmpty) { + t.Fatalf("an installer with no control-plane image gave %v, want ErrEmpty", err) + } + if !strings.Contains(err.Error(), "make bootstrap") { + t.Errorf("the refusal does not say how to build one that carries an image: %v", err) + } +} + +// Two different images cannot share an id, so an answer that is not the id asked about means the +// runtime is talking about something else. Reported rather than believed. +func TestARuntimeAnsweringAboutADifferentImageIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, _ []string) (string, error) { + return "sha256:" + strings.Repeat("9", 64) + "\n", nil + }} + if _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil { + t.Fatal("the runtime answered about a different image and it was accepted") + } +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go new file mode 100644 index 0000000..476a530 --- /dev/null +++ b/internal/bootstrap/preflight.go @@ -0,0 +1,211 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/image" + "github.com/novox/mesh-host/internal/profile" +) + +// DefaultRegistry is where an image reference that names no host comes from. +const DefaultRegistry = "registry-1.docker.io:443" + +// Preflight refuses early and plainly, and returns the bundle template it read. +// +// Everything here is a thing that will otherwise be discovered half way through: a machine with +// no runtime found after a bundle has been written, a template that does not parse found after an +// image has been loaded, a registry that cannot be reached found inside a `docker pull` that +// reports a network error and not a missing image. The order is cheapest first, so a mistake in +// what the installer was pointed at costs nothing to find. +func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte, error) { + // 1. Does this installer carry what it claims to? + // + // Asked before the machine is touched, for the same reason `mesh-host bundle` exists: a host + // that carries no substrate must say so when somebody asks, not on a first node + // (internal/bundle). An installer built without an image would otherwise get a machine as far + // as a running store and a running broker and stop. + if image.IsEmpty() { + return nil, image.ErrEmpty + } + saved, err := image.Saved() + if err != nil { + return nil, err + } + carriedID, err := image.ID(saved) + if err != nil { + return nil, err + } + say(fmt.Sprintf(" control plane %s carried (%s)", + firstOr(image.Tags(saved), "untagged"), carriedID)) + + // 2. Is the template there, and is it a substrate? + template, err := os.ReadFile(o.Template) + if err != nil { + return nil, fmt.Errorf( + "the bundle template could not be read: %w\n"+ + "It is what this machine will be asked to be, so there is nothing to do without "+ + "it. Point --bundle at one; mesh-host's examples/substrate-first-node.lock is "+ + "the shape", err) + } + // Parsed here as well as at the rewrite, because a template that is not a declaration should + // cost a second rather than an image load and a written file. + parsed, err := declaration.ParseFileTrusted(template) + if err != nil { + return nil, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + if _, err := controlPlaneIn(parsed); err != nil { + return nil, err + } + say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources))) + + // 3. Does a container runtime ANSWER? + // + // Not "is it installed" — novox/hq 04-ISSUES/007 is exactly that mistake, and the detector + // this uses is the one written for it: it asks the daemon for its server version, which fails + // when the daemon is down however complete the installation is. + // + // **Yes, the bundle installs the runtime itself**, and that is not a contradiction. The + // installer needs one BEFORE the apply, because the control plane's image is loaded into it + // first; the bundle still declares the package and the service because the host must own them + // and reassert them at every reconcile. So this is not a duplicate check — it is the one thing + // the bootstrap cannot bootstrap. + // + // Polled rather than asked once. A socket-activated daemon queued behind + // `network-online.target` is not absent, it is a few seconds away, and `docker load` against + // one blocks silently rather than failing (04-ISSUES/024). Waiting is the honest reading. + if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil { + return nil, err + } + + // 4. Can this machine reach what the bundle's images come from? + // + // Asked of the hosts the bundle actually names rather than of the internet in general. The + // mesh's own image is carried and needs nothing served — it is skipped here for exactly that + // reason. Everything else is somebody else's image at somebody else's registry, and a machine + // that cannot reach it fails inside a pull, which reports a network error where a person + // reads a missing image. + for _, host := range registriesIn(parsed) { + dialing, cancel := context.WithTimeout(ctx, o.Timeout) + err := d.Dial(dialing, host) + cancel() + if err != nil { + return nil, fmt.Errorf( + "this machine cannot reach %s, and the bundle's images are served from there: "+ + "%w\nThe apply would fail inside a pull, which says the wrong thing. Fix the "+ + "machine's network, or point the bundle at a registry it can reach", + host, err) + } + say(" reachable " + host) + } + return template, nil +} + +// waitForRuntime asks the runtime, repeatedly, until it answers or the wait runs out. +func waitForRuntime(ctx context.Context, run Runner, probe, wait time.Duration, say func(string)) error { + detector := containerRuntimeDetector(run) + + deadline := time.Now().Add(wait) + var last string + for { + probing, cancel := context.WithTimeout(ctx, probe) + verdict := detector.Detect(probing) + cancel() + if verdict.Present { + say(" container runtime " + verdict.Detail) + return nil + } + last = verdict.Detail + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(runtimeAskEvery): + } + } + return fmt.Errorf( + "this machine has no container runtime that answers, after waiting %s: %s\n"+ + "An installed package is not a capability (novox/hq 04-ISSUES/007) — the daemon was "+ + "asked and did not reply. Start it, then run this again; every step is idempotent", + wait, last) +} + +// runtimeAskEvery is how often the runtime is asked again while waiting for it. +var runtimeAskEvery = 2 * time.Second + +// containerRuntimeDetector is the host's OWN detector for a working runtime, not a second +// implementation of the same question. Two answers to "is there a container runtime here" is how +// the installer and the host come to disagree about a machine. +func containerRuntimeDetector(run Runner) profile.Detector { + for _, detector := range profile.Default(profile.Runner(run)) { + if detector.Name() == profile.CapContainerRuntime { + return detector + } + } + // Unreachable unless the host's own detector set loses its container runtime, which would be + // a change nobody would make on purpose — said rather than nil-dereferenced. + panic("the host detects no container runtime capability, and the installer needs that answer") +} + +// registriesIn is every host the bundle's images would be fetched from, without duplicates and in +// the order they appear. +func registriesIn(d *declaration.Declaration) []string { + var hosts []string + seen := map[string]bool{} + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok { + continue + } + host, served := registryOf(container.Image) + if !served || seen[host] { + continue + } + seen[host] = true + hosts = append(hosts, host) + } + return hosts +} + +// registryOf says where an image would be fetched from, and whether anything has to serve it. +// +// The second return is false for an image named by the digest of its own configuration: nothing +// serves those and nothing can (see `internal/declaration`'s checkImage). That is the whole reason +// the mesh's own control plane can be raised on a machine with no registry anywhere. +// +// The rule for the rest is the container runtime's own: the part before the first slash is a +// registry host if it looks like one — it has a dot, or a port, or it is `localhost` — and +// otherwise it is part of a repository name on the default registry. +func registryOf(reference string) (string, bool) { + if reference == "" || strings.HasPrefix(reference, "sha256:") { + return "", false + } + name := reference + if at := strings.Index(name, "@"); at >= 0 { + name = name[:at] + } + + first, _, hasPath := strings.Cut(name, "/") + if !hasPath || !(strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost") { + return DefaultRegistry, true + } + if !strings.Contains(first, ":") { + // A registry with no port is reached over HTTPS, which is where a pull would go. + return first + ":443", true + } + return first, true +} + +func firstOr(values []string, fallback string) string { + if len(values) == 0 || strings.TrimSpace(values[0]) == "" { + return fallback + } + return values[0] +} diff --git a/internal/bootstrap/preflight_test.go b/internal/bootstrap/preflight_test.go new file mode 100644 index 0000000..627dce0 --- /dev/null +++ b/internal/bootstrap/preflight_test.go @@ -0,0 +1,126 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// An installed package is not a capability (novox/hq 04-ISSUES/007). The daemon is asked, and a +// machine where it does not answer is refused before anything is loaded, written or applied. +// +// The refusal has to be plain, because the person reading it is standing in front of a machine +// that will not work: it says what was asked, what came back, that re-running is safe, and names +// the record that explains why an installed docker is not enough. +func TestPreflightRefusesPlainlyWhenTheRuntimeDoesNotAnswer(t *testing.T) { + silent := func(context.Context, string, ...string) (string, error) { + return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock") + } + + // No wait, so this is one attempt: what is being tested is the refusal, not the patience. + err := waitForRuntime(context.Background(), silent, time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a machine whose container runtime does not answer was accepted") + } + for _, wanted := range []string{ + "no container runtime that answers", + "Cannot connect to the Docker daemon", + "04-ISSUES/007", + "idempotent", + } { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a runtime that is merely slow to start is waited for rather than refused. +// +// A socket-activated daemon queued behind the network is not absent, it is a few seconds away. +// Refusing on the first attempt would make a correct bootstrap fail for being observed too early — +// and `docker load` against such a daemon blocks silently rather than failing, which is how one +// became a 35-minute silence (04-ISSUES/024). +func TestARuntimeThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := runtimeAskEvery + runtimeAskEvery = time.Millisecond + defer func() { runtimeAskEvery = previous }() + + attempts := 0 + slow := func(context.Context, string, ...string) (string, error) { + attempts++ + if attempts < 3 { + return "", errors.New("Cannot connect to the Docker daemon") + } + return "27.0.3\n", nil + } + + var said []string + if err := waitForRuntime(context.Background(), slow, time.Second, time.Second, + func(line string) { said = append(said, line) }); err != nil { + t.Fatalf("a runtime that answered on the third ask was refused: %v", err) + } + if attempts != 3 { + t.Errorf("the runtime was asked %d time(s)", attempts) + } + if !strings.Contains(strings.Join(said, "\n"), "27.0.3") { + t.Errorf("the version the daemon reported was not said back: %v", said) + } +} + +// What has to be reachable is what the bundle actually names, not "the internet". +// +// The mesh's own image is carried and nothing serves it, so asking a registry about it would be +// asking a question with no answer — which is the whole point of naming an image by the digest of +// its own configuration. +func TestOnlyTheRegistriesTheBundleNamesAreAskedAbout(t *testing.T) { + parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"}, + {"id":"broker","type":"container","name":"mesh-broker","image":"192.0.2.250:5000/lavinmq@sha256:` + + strings.Repeat("8", 64) + `"}, + {"id":"control-plane","type":"container","name":"mesh-control","image":"` + held + `"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + got := registriesIn(parsed) + want := []string{DefaultRegistry, "192.0.2.250:5000"} + if len(got) != len(want) { + t.Fatalf("asked about %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("asked about %v, want %v", got, want) + } + } +} + +func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) { + // The container runtime's own rule: the part before the first slash is a registry host if it + // has a dot, a port, or is localhost. Getting this wrong means dialling a hostname that is + // really the first half of a repository name, and refusing a machine that is fine. + for _, c := range []struct { + reference string + host string + served bool + }{ + {"postgres@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"cloudamqp/lavinmq@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"192.0.2.250:5000/postgres@sha256:" + strings.Repeat("a", 64), "192.0.2.250:5000", true}, + {"localhost/mesh-control@sha256:" + strings.Repeat("a", 64), "localhost:443", true}, + {"registry.example.com/a/b@sha256:" + strings.Repeat("a", 64), "registry.example.com:443", true}, + // Held by this machine. Nothing serves it, and nothing can. + {"sha256:" + strings.Repeat("a", 64), "", false}, + {"", "", false}, + } { + host, served := registryOf(c.reference) + if host != c.host || served != c.served { + t.Errorf("%q → (%q, %v), want (%q, %v)", c.reference, host, served, c.host, c.served) + } + } +} diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go new file mode 100644 index 0000000..2cdcde5 --- /dev/null +++ b/internal/bootstrap/rewrite.go @@ -0,0 +1,241 @@ +package bootstrap + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// ControlPlaneID is the resource the installer replaces the image of. +// +// A resource id rather than a container name or a guess at the image, because the id is the one +// thing a declaration promises is stable — it is what lets the store say *this is the same +// resource I applied last time* (`internal/declaration`, Resource.Identity). A bundle that does not +// name one is refused rather than applied without a control plane, which would raise a store and a +// broker and no mesh. +const ControlPlaneID = "control-plane" + +// brokerAddressVar is what a token tells an enrolling node to dial. +// +// Not rewritten here — see the note in Rewrite — but reported, because it is the field most likely +// to be wrong on a machine that is not the one the template was written for, and it is wrong in a +// way nothing notices until a second node tries to join. +const brokerAddressVar = "MESH_BROKER_ADDRESS" + +// Rewritten is the bundle this machine will apply, and what was done to produce it. +type Rewritten struct { + // Bundle is the produced file's bytes — the template with one image reference replaced, + // comments and all. + Bundle []byte + // Declaration is that bundle, parsed. Carried so the apply and the verify are talking about + // the same document rather than each re-reading the file and hoping. + Declaration *declaration.Declaration + + // Was is the image the template named the control plane by; Now is the one it names it by. + Was string + Now string + // Places is how many times that reference appeared, and therefore how many were replaced. + Places int + // Changed is false when the template already named this image — a re-run on a bundle this + // installer produced earlier. + Changed bool + + // Kept is every other container image, unchanged, as " ". Reported rather than + // assumed: "postgres was left alone" is a claim, and this is the evidence for it. + Kept []string + + // Resources is how many things the produced bundle asks for. + Resources int + // BrokerAddress is what the control plane will tell enrolling nodes to dial, or empty. + BrokerAddress string +} + +// Rewrite produces the bundle this machine will apply from the template it was given. +// +// **One substitution, and it is textual.** The control plane's image becomes the id of the image +// this machine now holds; nothing else changes. Textual rather than parse-and-re-serialise because +// the produced file has to be *read* — a person getting a machine working must be able to open it, +// see the substrate they recognise, and see exactly one thing different. Re-serialising a parsed +// declaration would drop every comment in the template, and those comments are where the reasons +// live. +// +// **Every place that reference appears, not only the container.** The bundle names the control +// plane's image twice: once as the container that runs `serve`, and once inside the action that +// runs `migrate` to create the contexts' schemas. Replacing only the container would leave the +// migration pointing at an image no registry serves, and the apply would fail in the middle — +// after the store is up, before the broker. They are one image and they move together. +// +// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references +// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is +// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other +// container's image is compared against what it was. +// +// **What this deliberately does NOT rewrite:** MESH_BROKER_ADDRESS, the endpoint every enrolment +// token will carry. It differs per machine and it is silently fatal when wrong — a node enrols +// against a dead address and nothing complains until it fails to come back. It belongs to the +// enrolment stage, which is not built yet, so this reports it loudly and leaves it alone rather +// than guessing an address for a machine it has not been told about. +func Rewrite(template []byte, imageID string) (Rewritten, error) { + if !isImageID(imageID) { + return Rewritten{}, fmt.Errorf( + "%q is not an image id. The control plane is named by the digest of its own "+ + "configuration — sha256: and sixty-four hex characters — because nothing serves "+ + "it and there is no manifest digest to use instead", imageID) + } + + before, err := declaration.ParseFileTrusted(template) + if err != nil { + return Rewritten{}, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + control, err := controlPlaneIn(before) + if err != nil { + return Rewritten{}, err + } + + out := Rewritten{Was: control.Image, Now: imageID, BrokerAddress: control.Env[brokerAddressVar]} + + occurrences := bytes.Count(template, []byte(control.Image)) + if occurrences == 0 { + // The parser found the image and the bytes do not contain it, which means the two are + // reading different things. Refused rather than replaced-zero-times-and-reported-success. + return Rewritten{}, fmt.Errorf( + "the control plane's image is %q according to the parsed template, and that text is "+ + "not in the file. Nothing was rewritten", control.Image) + } + out.Places = occurrences + + switch { + case control.Image == imageID: + // Already this image. The idempotent case, and the one that happens whenever somebody + // re-runs the installer against a bundle it produced earlier. + out.Bundle = template + default: + out.Bundle = bytes.ReplaceAll(template, []byte(control.Image), []byte(imageID)) + out.Changed = true + } + + // Read back, on the bytes that will actually be applied. Everything above is an intention + // until the produced file is parsed and asked what it says. + after, err := declaration.ParseFileTrusted(out.Bundle) + if err != nil { + return Rewritten{}, fmt.Errorf( + "the bundle this produced is not a declaration, so the substitution broke it: %w", err) + } + out.Declaration, out.Resources = after, len(after.Resources) + + produced, err := controlPlaneIn(after) + if err != nil { + return Rewritten{}, err + } + if produced.Image != imageID { + return Rewritten{}, fmt.Errorf( + "the produced bundle still names the control plane %q, not %q", produced.Image, imageID) + } + + // And nothing else moved. A substitution on text can in principle catch more than it was + // aimed at, and "postgres was left exactly as it was" is the claim this checks rather than + // asserts. + was := containerImages(before) + for id, image := range containerImages(after) { + if id == ControlPlaneID { + continue + } + if was[id] != image { + return Rewritten{}, fmt.Errorf( + "rewriting the control plane's image also changed %q, from %q to %q. Only the "+ + "mesh's own image may move; everything else is somebody else's image at "+ + "somebody else's registry", id, was[id], image) + } + out.Kept = append(out.Kept, fmt.Sprintf("%s %s", id, image)) + } + sortStrings(out.Kept) + return out, nil +} + +// controlPlaneIn finds the container this installer replaces the image of. +func controlPlaneIn(d *declaration.Declaration) (*declaration.Container, error) { + for _, r := range d.Resources { + if r.Identity() != ControlPlaneID { + continue + } + container, ok := r.(*declaration.Container) + if !ok { + return nil, fmt.Errorf( + "this bundle's %q is a %s, and the control plane has to be a container for its "+ + "image to be named. Nothing was rewritten", ControlPlaneID, r.Kind()) + } + return container, nil + } + return nil, fmt.Errorf( + "this bundle names no %q, so there is no control plane to give this machine's image to. "+ + "A substrate without one raises a store and a broker and no mesh. It declares: %s", + ControlPlaneID, strings.Join(identities(d), ", ")) +} + +func containerImages(d *declaration.Declaration) map[string]string { + images := map[string]string{} + for _, r := range d.Resources { + if container, ok := r.(*declaration.Container); ok { + images[container.ID] = container.Image + } + } + return images +} + +func identities(d *declaration.Declaration) []string { + var ids []string + for _, r := range d.Resources { + ids = append(ids, r.Identity()) + } + return ids +} + +// isImageID is the same shape `internal/declaration` accepts for an image the machine holds. Asked +// here as well so the refusal names the installer's own mistake, rather than surfacing as a +// declaration refusal about a bundle this program wrote. +func isImageID(s string) bool { + const prefix = "sha256:" + if !strings.HasPrefix(s, prefix) || len(s) != len(prefix)+64 { + return false + } + for _, c := range s[len(prefix):] { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func sortStrings(values []string) { + for i := 1; i < len(values); i++ { + for j := i; j > 0 && values[j] < values[j-1]; j-- { + values[j], values[j-1] = values[j-1], values[j] + } + } +} + +// writeBundleFile puts the produced bundle where a person can read it, creating the directory it +// lives in. +// +// 0644, and that is deliberate: this file names an image and describes a substrate, and it holds +// the bootstrap credentials the template happens to carry — which are the same ones anybody can +// read in the template itself. It is meant to be read. What must not be world-readable is the +// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`). +func writeBundleFile(path string, content []byte) error { + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err) + } + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf( + "cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+ + "applying something nobody can read afterwards is how a machine becomes a mystery", + path, err) + } + return nil +} diff --git a/internal/bootstrap/rewrite_test.go b/internal/bootstrap/rewrite_test.go new file mode 100644 index 0000000..11798ae --- /dev/null +++ b/internal/bootstrap/rewrite_test.go @@ -0,0 +1,223 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +const ( + held = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + otherHeld = "sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +// theRealBundle is this repository's own substrate example, used rather than a fixture. +// +// A fixture would agree with whatever this code does. The example is what an installer is actually +// pointed at, it names the control plane twice, and it is the file that changes when the substrate +// changes — so a rewrite that stops working on it is a rewrite that has stopped working. +func theRealBundle(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile("../../examples/substrate-first-node.lock") + if err != nil { + t.Fatalf("reading the substrate example: %v", err) + } + return raw +} + +func TestTheControlPlaneIsNamedByTheImageThisMachineHolds(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if !out.Changed { + t.Error("the rewrite reported nothing changed, and the template named a registry image") + } + control, err := controlPlaneIn(out.Declaration) + if err != nil { + t.Fatal(err) + } + if control.Image != held { + t.Errorf("the control plane is %q, want %q", control.Image, held) + } +} + +// **Every place the bundle names that image, not only the container.** +// +// The substrate names the control plane's image twice: the container that runs `serve`, and the +// action that runs `migrate` to create the contexts' schemas. Rewriting only the container leaves +// the migration pointing at an image no registry serves, and the apply dies in the middle — after +// the store is up and before the broker. This is the test that would have caught that. +func TestEveryPlaceTheBundleNamesTheControlPlaneIsRewritten(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.Places < 2 { + t.Fatalf("the control plane's image was found in %d place(s); the substrate names it in "+ + "the container AND in the migration action", out.Places) + } + if remaining := strings.Count(string(out.Bundle), out.Was); remaining != 0 { + t.Errorf("the produced bundle still names %q in %d place(s)", out.Was, remaining) + } + if got := strings.Count(string(out.Bundle), held); got != out.Places { + t.Errorf("the produced bundle names the held image %d time(s), and %d were replaced", + got, out.Places) + } +} + +// Third-party images are somebody else's, at somebody else's registry, and the installer has no +// business touching them (novox/hq ADR 0006). +func TestPostgresAndTheBrokerAreLeftExactlyAsTheyWere(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + + produced := containerImages(out.Declaration) + for _, id := range []string{"store", "broker"} { + image, named := produced[id] + if !named { + t.Fatalf("the substrate example no longer declares a %q container", id) + } + // Compared against the template's own text rather than against an expectation written + // here: what is being defended is "unchanged", and the template is the only thing that + // knows what it said. + if !strings.Contains(string(template), `"image": "`+image+`"`) { + t.Errorf("%s is now %q, which the template does not say", id, image) + } + if strings.HasPrefix(image, "sha256:") { + t.Errorf("%s was rewritten to an image this machine holds, and nothing holds it", id) + } + } + + // And the claim in the report is the same claim, so a person reading it is reading evidence. + if len(out.Kept) != 2 { + t.Errorf("the rewrite reports %d untouched image(s): %v", len(out.Kept), out.Kept) + } +} + +// A bundle with no control plane raises a store and a broker and no mesh. Refused, because +// applying it would succeed and leave a machine that looks bootstrapped. +func TestABundleThatNamesNoControlPlaneIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"} + ]}`) + + _, err := Rewrite(template, held) + if err == nil { + t.Fatal("a bundle with no control plane was rewritten and would have been applied") + } + // The refusal has to be actionable: it says what the bundle DID declare, so somebody can see + // they pointed it at the wrong file or misspelled the id. + if !strings.Contains(err.Error(), ControlPlaneID) || !strings.Contains(err.Error(), "store") { + t.Errorf("the refusal names neither what was wanted nor what was there: %v", err) + } +} + +func TestAControlPlaneThatIsNotAContainerIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"control-plane","type":"package","package":"mesh-control"} + ]}`) + if _, err := Rewrite(template, held); err == nil { + t.Fatal("a control plane declared as a package was accepted, and a package has no image") + } +} + +// Idempotence. This program is run over and over while somebody gets a machine working, and the +// second run must be able to say the bundle already names this image rather than reporting a +// rewrite it did not perform. +func TestRewritingABundleThatAlreadyNamesTheImageChangesNothing(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + second, err := Rewrite(first.Bundle, held) + if err != nil { + t.Fatal(err) + } + if second.Changed { + t.Error("re-running the rewrite reported a change, and the image was already the one held") + } + if string(second.Bundle) != string(first.Bundle) { + t.Error("re-running the rewrite produced different bytes") + } + if second.Was != held { + t.Errorf("the second run reports it replaced %q; it replaced nothing", second.Was) + } +} + +// A DIFFERENT image, though, must move — the ordinary case of a new control plane being installed +// over an old one. "Already correct" must not be the same code path as "already ran". +func TestANewImageReplacesAnOlderHeldOne(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + second, err := Rewrite(first.Bundle, otherHeld) + if err != nil { + t.Fatal(err) + } + if !second.Changed || second.Was != held || second.Now != otherHeld { + t.Errorf("a new image did not replace the old one: changed=%v was=%q now=%q", + second.Changed, second.Was, second.Now) + } +} + +// The produced bundle is meant to be READ. Re-serialising a parsed declaration would drop every +// comment in the template, and the substrate example is mostly comments — each one recording why a +// resource is the way it is, several of them paid for in the lab. +func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + const remembered = "NAMED VOLUME" + if !strings.Contains(string(out.Bundle), remembered) { + t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered) + } +} + +// The installer must refuse its own bad input in its own words, rather than writing a bundle and +// letting the host refuse a declaration somebody did not write. +func TestSomethingThatIsNotAnImageIdIsRefused(t *testing.T) { + for _, bad := range []string{ + "", + "mesh-control:latest", + "sha256:abc", + "sha256:" + strings.Repeat("1", 63), + "sha256:" + strings.Repeat("g", 64), + "mesh-control@sha256:" + strings.Repeat("1", 64), + } { + if _, err := Rewrite(theRealBundle(t), bad); err == nil { + t.Errorf("image id %q was accepted", bad) + } + } +} + +// The address every enrolment token will carry is reported and never invented. It differs per +// machine and it is silently fatal when wrong: a node enrols against a dead address and nothing +// says so until it fails to come back. +func TestTheAddressNodesWillDialIsReportedAndNotRewritten(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.BrokerAddress == "" { + t.Fatal("the substrate example no longer says what address enrolling nodes will dial") + } + if !strings.Contains(string(out.Bundle), out.BrokerAddress) { + t.Errorf("the produced bundle no longer carries %q — it was rewritten, and nothing here "+ + "knows this machine's address", out.BrokerAddress) + } +} diff --git a/internal/bootstrap/verify.go b/internal/bootstrap/verify.go new file mode 100644 index 0000000..4bed6fe --- /dev/null +++ b/internal/bootstrap/verify.go @@ -0,0 +1,167 @@ +package bootstrap + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// controlPlaneBinary is where the control plane's program lives in its own image. +// +// A path rather than a shell command, because the image is `FROM scratch` and holds one static +// binary and nothing else — no shell to invoke, nothing to interpret a command line +// (mesh-control's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer +// carries, which is why the path can be written down here. +const controlPlaneBinary = "/mesh-control" + +// answerEvery is how often the control plane is asked again while it is starting. +var answerEvery = 2 * time.Second + +// Verified is what the substrate was found to be. +type Verified struct { + // Running is every long-running container the bundle declares, confirmed up. + Running []string + // Answered is the control plane's own first words back, so the report shows the reply rather + // than asserting there was one. + Answered string +} + +// Verify proves the substrate is up and the control plane replies. +// +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once: a runtime reports a container running from the moment the +// process starts, which is before it has opened a database, before it has read its configuration, +// and before it has failed to. So the containers are checked, and then the program inside one of +// them is asked a question and has to answer it. +// +// The question is `status`, and it is chosen rather than convenient: answering it means the +// control plane opened all three of its stores from the environment the bundle gave it. A reply +// therefore proves the image runs, that `network: host` really does reach the store on this +// machine, and that the contexts' schemas migrated — the three things the steps before this were +// for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server. +// +// It waits. A control plane that is not answering yet and a control plane that will never answer +// look identical for the first few seconds, and refusing on the first attempt would make a correct +// bootstrap fail for being observed too early. +func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration, + say func(string)) (Verified, error) { + + var out Verified + + control, err := controlPlaneIn(d) + if err != nil { + return out, err + } + + for _, container := range longRunning(d) { + state, err := containerRunning(ctx, run, probe, container) + if err != nil { + return out, err + } + if !state.running { + return out, fmt.Errorf( + "the container %q is %s, not running.\n"+ + "The apply reported success, so it was created — what it did afterwards is "+ + "in `docker logs %s`", container, state.status, container) + } + out.Running = append(out.Running, container) + say(" running " + container) + } + + answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say) + if err != nil { + return out, err + } + out.Answered = answer + return out, nil +} + +func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration, + container string, say func(string)) (string, error) { + + deadline := time.Now().Add(wait) + var last error + for { + asking, cancel := context.WithTimeout(ctx, probe) + out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status") + cancel() + + answer := strings.TrimSpace(firstLineOf(out)) + switch { + case err != nil: + last = err + case answer == "": + // Exit zero and nothing said. Treated as no answer rather than as success: a program + // that returns silence is not one that has been asked anything. + last = fmt.Errorf("it exited without saying anything") + default: + say(" replies " + container + ": " + answer) + return answer, nil + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } + return "", fmt.Errorf( + "the container %q is running and the control plane in it does not answer, after waiting "+ + "%s: %v\n"+ + "Running is not replying. `status` opens this mesh's three stores, so what failed is "+ + "most likely the store or the schemas rather than the control plane itself — "+ + "`docker logs %s` says which", container, wait, last, container) +} + +type containerState struct { + running bool + status string +} + +func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) { + asking, cancel := context.WithTimeout(ctx, probe) + defer cancel() + + // Both facts in one answer, so a container that is not running is reported with what it IS + // rather than with the absence of what it should be. + out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name) + if err != nil { + return containerState{}, fmt.Errorf( + "the container %q is not there at all, and the apply reported it applied: %w", name, err) + } + running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ") + if status == "" { + status = "in a state the runtime did not name" + } + return containerState{running: running == "true", status: status}, nil +} + +// longRunning is every container the bundle expects to still be there afterwards. +// +// A run-once step has exited by design and a scheduled step has deliberately never been started +// (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the +// substrate to be something other than what it declared. +func longRunning(d *declaration.Declaration) []string { + var names []string + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok || container.RunOnce || container.Schedule != "" { + continue + } + names = append(names, container.Name) + } + return names +} + +func firstLineOf(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go new file mode 100644 index 0000000..45eb857 --- /dev/null +++ b/internal/bootstrap/verify_test.go @@ -0,0 +1,168 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +func substrate(t *testing.T) *declaration.Declaration { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Declaration +} + +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once. A runtime reports a container running from the moment its +// process starts — before it has opened a database, and before it has failed to. +func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + switch args[0] { + case "inspect": + return "true running\n", nil + case "exec": + // Up, and saying nothing. The program inside is not answering. + return "", errors.New("exit status 1") + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("every container was running, nothing answered, and the substrate was reported up") + } + for _, wanted := range []string{"mesh-control", "Running is not replying", "docker logs"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the failure does not mention %q:\n%v", wanted, err) + } + } +} + +// Exit zero and silence is not an answer either. A program that returns nothing has not been asked +// anything, and treating it as success is the same fault one level down. +func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return " \n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err == nil { + t.Fatal("a control plane that exited zero without saying anything was accepted") + } +} + +// The substrate answering is the whole point, and what it said is reported rather than asserted. +func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node, 0 waiting\n", nil + }} + + verified, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err != nil { + t.Fatal(err) + } + // Three long-running containers: the store, the broker and the control plane. The run-once and + // scheduled shapes are excluded on purpose — a step that has exited is not a fault. + want := []string{"mesh-store", "mesh-broker", "mesh-control"} + if len(verified.Running) != len(want) { + t.Fatalf("confirmed %v running, want %v", verified.Running, want) + } + for i := range want { + if verified.Running[i] != want[i] { + t.Errorf("confirmed %v running, want %v", verified.Running, want) + } + } + if verified.Answered != "1 node, 0 waiting" { + t.Errorf("the control plane's reply is reported as %q", verified.Answered) + } +} + +// A control plane that is still opening its stores is waited for, not refused. Refusing on the +// first attempt would make a correct bootstrap fail for being observed too early. +func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + attempts := 0 + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + attempts++ + if attempts < 3 { + return "", errors.New("exit status 1") + } + return "1 node\n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, time.Second, func(string) {}); err != nil { + t.Fatalf("a control plane that answered on the third ask was refused: %v", err) + } +} + +// A container that exited is named with what it IS, so somebody can go and read its logs rather +// than being told only that something is not what it should be. +func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" { + return "false exited\n", nil + } + if args[0] == "inspect" { + return "true running\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a container that had exited was reported as part of a running substrate") + } + if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { + t.Errorf("the failure does not say which container is in what state: %v", err) + } +} + +// The control plane is asked by running the binary in its own image directly, because the image is +// `FROM scratch` and has no shell for a command line to be interpreted by. +func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node\n", nil + }} + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err != nil { + t.Fatal(err) + } + if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the control plane was never asked anything: %v", runtime.commands) + } +} diff --git a/internal/image/control-plane.tar b/internal/image/control-plane.tar new file mode 100644 index 0000000..e26e4f9 --- /dev/null +++ b/internal/image/control-plane.tar @@ -0,0 +1,8 @@ +This is not a saved image. It is the placeholder that keeps this repository buildable. + +A release build replaces this file with the output of `docker save` and puts it back afterwards: + + make bootstrap IMAGE=mesh-control: + +An installer built with this file present carries no control plane, and says so in preflight +rather than getting a machine part-way to being a mesh and stopping. diff --git a/internal/image/image.go b/internal/image/image.go new file mode 100644 index 0000000..5e94196 --- /dev/null +++ b/internal/image/image.go @@ -0,0 +1,173 @@ +// Package image is the control plane's image, carried inside the installer. +// +// **Carried rather than built, and that is not an optimisation.** The mesh's forge runs on the +// mesh. A bootstrap that needed the control plane's source in order to build it would need a +// forge to fetch that source from, and the forge is one of the things the mesh raises — so the +// mesh would be required in order to raise the mesh. Embedding the image breaks that cycle the +// same way `internal/bundle` breaks it for the declaration: the installer arrives holding +// everything a bare machine has to be given, and a bare machine is given one file +// (novox/hq ADR 0005). +// +// It is also the rule the rest of tier 0 already follows. Nobody compiles `mesh-host` on the +// machine it will run on; the binary is put there. The image it raises arrives the same way. +// +// What is embedded is the output of `docker save` — a tar holding the image's config, its layers +// and a `manifest.json` naming them. The control plane's image is `FROM scratch` with one static +// binary in it (novox/hq ADR 0006), so this is tens of megabytes rather than hundreds. +package image + +import ( + "archive/tar" + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" +) + +// The saved image, replaced at release time by `make bootstrap`. +// +// What is committed here is a placeholder, for the same reason `internal/bundle` commits locks +// that are only comments: `go:embed` refuses to compile against a file that is not there, so a +// checkout with nothing embedded would not build at all — and someone reading this repository or +// running `go test ./...` would meet a compile error instead of a program. The placeholder keeps +// the tree buildable and makes the absence a thing the installer *says*, at the earliest moment +// it can, rather than a thing a compiler says to the wrong person. +// +// It is small and it is committed. A saved image is not, and `make bootstrap` puts one here for +// the length of one build and then puts the placeholder back — exactly what `make host` does with +// the bundle it embeds. +// +//go:embed control-plane.tar +var saved []byte + +// ErrEmpty means this installer carries no control-plane image. +// +// A separate error rather than a message, so the caller can refuse in preflight — before a +// machine has been touched — instead of discovering it at the load, after the runtime has been +// probed and a bundle has been written. +var ErrEmpty = errors.New( + "this mesh-bootstrap carries no control-plane image, so it cannot raise a mesh. A release " + + "build embeds one: `make bootstrap IMAGE=` in the mesh-host repository, where " + + " is a control-plane image already built from the mesh-control source") + +// IsEmpty reports whether anything was built in. +// +// It asks whether the bytes are a tar rather than comparing them against the placeholder's text, +// because the question that matters is "can this be loaded", and a truncated or corrupted embed +// answers no to that while matching no placeholder. A tar's first header carries the string +// `ustar` at offset 257 and nothing else does by accident. +func IsEmpty() bool { + const magicAt, magic = 257, "ustar" + return len(saved) < magicAt+len(magic) || string(saved[magicAt:magicAt+len(magic)]) != magic +} + +// Saved returns the embedded tar, for loading into a container runtime. +func Saved() ([]byte, error) { + if IsEmpty() { + return nil, ErrEmpty + } + return saved, nil +} + +// manifestEntry is the part of a saved image's `manifest.json` this needs. +// +// One field. The layers are the runtime's business and the repository tags are decoration — what +// is wanted is the config, because the digest of the config IS the image id. +type manifestEntry struct { + Config string `json:"Config"` + RepoTags []string `json:"RepoTags"` +} + +// ID is the image id the runtime will give this image once it is loaded, read out of the tar. +// +// **Read here rather than parsed out of what `docker load` prints.** The load prints a sentence +// for a person — `Loaded image: name:tag` or `Loaded image ID: sha256:…`, depending on whether the +// image was saved with a tag — and a program that scraped it would be depending on which of those +// a particular runtime version chose. The id is a fact about the file, available before the +// runtime is asked anything, which is also what makes the load idempotent: the installer can ask +// whether the machine already holds THIS image before loading it. +// +// An image id is the sha256 of the image's configuration document (novox/hq ADR 0006, and see +// `internal/declaration`'s checkImage). `manifest.json` names that document by its digest — as +// `<64hex>.json` in the older layout and `blobs/sha256/<64hex>` in the OCI one — so both forms +// reduce to the same sixty-four characters. +func ID(saved []byte) (string, error) { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return "", err + } + + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil { + return "", fmt.Errorf( + "the carried image has a manifest.json this cannot read, so the image it holds "+ + "cannot be named: %w", err) + } + // One image, deliberately. A tar holding several would leave the installer choosing which + // one is the control plane, and a bootstrap must not be the thing that guesses. + if len(entries) != 1 { + return "", fmt.Errorf( + "the carried image holds %d images, and the installer raises exactly one control "+ + "plane. Save a single image: `docker save --output … `", len(entries)) + } + + digest := strings.TrimSuffix(path.Base(entries[0].Config), ".json") + if !isSHA256(digest) { + return "", fmt.Errorf( + "the carried image names its configuration %q, which is not a sha256 digest. An "+ + "image id is the digest of that configuration, so there is nothing to call this "+ + "image", entries[0].Config) + } + return "sha256:" + digest, nil +} + +// Tags is what the saved image was called when it was saved, for a person reading a report. +// +// Decoration, and said so: the installer names the image by its id everywhere it matters, because +// a tag is exactly what a pinned bundle may not rely on (novox/hq ADR 0006). This is here so a +// report can say which build somebody embedded, which is otherwise sixty-four characters of hex. +func Tags(saved []byte) []string { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return nil + } + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil || len(entries) == 0 { + return nil + } + return entries[0].RepoTags +} + +func fileFromTar(archive []byte, want string) ([]byte, error) { + reader := tar.NewReader(bytes.NewReader(archive)) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf( + "the carried image has no %s, so it is not something `docker save` produced. "+ + "Embed the output of `docker save`, not a layer or a build context", want) + } + if err != nil { + return nil, fmt.Errorf("the carried image cannot be read as a tar: %w", err) + } + if path.Clean(header.Name) == want { + return io.ReadAll(reader) + } + } +} + +func isSHA256(s string) bool { + if len(s) != 64 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/image/image_test.go b/internal/image/image_test.go new file mode 100644 index 0000000..74970db --- /dev/null +++ b/internal/image/image_test.go @@ -0,0 +1,152 @@ +package image + +import ( + "archive/tar" + "bytes" + "encoding/json" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +// savedImage builds what `docker save` produces, as far as this package reads it. +func savedImage(t *testing.T, files map[string]string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + for name, content := range files { + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))} + if err := writer.WriteHeader(header); err != nil { + t.Fatalf("building the fixture: %v", err) + } + if _, err := writer.Write([]byte(content)); err != nil { + t.Fatalf("building the fixture: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("building the fixture: %v", err) + } + return buffer.Bytes() +} + +func manifest(t *testing.T, config string, tags ...string) string { + t.Helper() + raw, err := json.Marshal([]manifestEntry{{Config: config, RepoTags: tags}}) + if err != nil { + t.Fatalf("building the fixture: %v", err) + } + return string(raw) +} + +// The image id is read from the FILE, before any runtime is asked anything. +// +// That is what makes the load idempotent: knowing the id in advance lets the installer ask "do you +// already hold exactly this" instead of loading and then finding out. Scraping it from what +// `docker load` prints would only be possible after loading, so the second run of an installer +// would load again every time and be unable to say it had not. +func TestTheImageIdIsReadFromTheSavedFile(t *testing.T) { + digest := strings.Repeat("a", 64) + // Both layouts `docker save` has used. The older one names the config `.json`; the OCI + // one names it `blobs/sha256/`. They carry the same sixty-four characters, and a + // reader that understood only one would work until somebody upgraded their runtime. + for _, config := range []string{digest + ".json", "blobs/sha256/" + digest} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + id, err := ID(saved) + if err != nil { + t.Fatalf("config %q: %v", config, err) + } + if id != "sha256:"+digest { + t.Errorf("config %q gave id %q, want sha256:%s", config, id, digest) + } + } +} + +func TestTheSavedTagsAreReadForAPersonToRecognise(t *testing.T) { + saved := savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("b", 64)+".json", "mesh-control:v1"), + }) + got := Tags(saved) + if len(got) != 1 || got[0] != "mesh-control:v1" { + t.Errorf("tags = %v, want [mesh-control:v1]", got) + } +} + +// A tar that is not a saved image is refused with what is wrong, not with a nil id. +// +// The installer names the control plane by this id in the bundle it writes. An id it could not +// read, treated as empty, would produce a bundle naming nothing — refused by the host two steps +// later, with a message about a declaration rather than about what somebody embedded. +func TestSomethingThatIsNotASavedImageIsRefused(t *testing.T) { + notAnImage := savedImage(t, map[string]string{"hello": "world"}) + if _, err := ID(notAnImage); err == nil { + t.Error("a tar with no manifest.json was accepted as a saved image") + } else if !strings.Contains(err.Error(), "docker save") { + t.Errorf("the refusal does not say what to embed instead: %v", err) + } + + if _, err := ID([]byte("this is not a tar at all")); err == nil { + t.Error("bytes that are not a tar were accepted") + } +} + +// Exactly one image. A bootstrap that chose between several would be the thing that guesses which +// one is the control plane, and it would guess right until the day somebody saved two. +func TestATarHoldingSeveralImagesIsRefused(t *testing.T) { + entries, err := json.Marshal([]manifestEntry{ + {Config: strings.Repeat("a", 64) + ".json"}, + {Config: strings.Repeat("b", 64) + ".json"}, + }) + if err != nil { + t.Fatal(err) + } + saved := savedImage(t, map[string]string{"manifest.json": string(entries)}) + if _, err := ID(saved); err == nil { + t.Error("a tar holding two images was accepted") + } +} + +// An id is a digest or it is nothing. A truncated one names several images, and which one ran +// would be whichever the runtime matched first — the same reasoning `internal/declaration` gives +// for refusing a short image reference. +func TestAConfigThatIsNotADigestIsRefused(t *testing.T) { + for _, config := range []string{"config.json", "abc.json", "blobs/sha256/" + strings.Repeat("a", 63)} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + if _, err := ID(saved); err == nil { + t.Errorf("config %q was accepted and is not a digest", config) + } + } +} + +// The committed placeholder must read as "carries nothing", so an installer built from a plain +// checkout says so in preflight rather than getting a machine as far as a running store and +// stopping. This is the same guarantee `internal/bundle` makes about a lock file of only comments. +func TestAnInstallerBuiltFromAPlainCheckoutCarriesNothing(t *testing.T) { + if !IsEmpty() { + // Not a failure of this checkout: `make bootstrap` embeds a real image and puts the + // placeholder back, so a real image here means a build was interrupted. + t.Skip("this checkout has a saved image embedded, so there is no placeholder to check") + } + if _, err := Saved(); err == nil { + t.Fatal("an installer carrying only the placeholder reported it carries an image") + } +} + +// And "empty" is decided by whether the bytes could be loaded, not by matching the placeholder's +// text. A truncated or corrupted embed is equally unloadable and equally worth refusing early. +func TestEmptyMeansUnloadableRatherThanEqualToThePlaceholder(t *testing.T) { + restore := saved + defer func() { saved = restore }() + + saved = []byte("half a tar, cut off") + if !IsEmpty() { + t.Error("bytes that are not a tar were reported as a carried image") + } + + saved = savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("c", 64)+".json"), + }) + if IsEmpty() { + t.Error("a real saved image was reported as no image at all") + } +}