novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mesh-bootstrap: the installer, and the two things its first real run found #8

Merged
jschoubben merged 9 commits from fix/bootstrap-first-run into main 2026-09-11 19:56:57 +00:00
Conversation 0 Commits 9 Files Changed 33
+6282 -5
33 changed files with 6282 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -1,2 +1,6 @@
/mesh-host
/mesh-bootstrap
/dist/
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
# interrupted release build cannot commit a twenty-megabyte tar by accident.
/internal/image/control-plane.tar.placeholder
Makefile
+41 -2
View File
@@ -7,7 +7,7 @@ LDFLAGS := -s -w -X main.builtFor=$(SYSTEM) -X main.version=$(VERSION)
# a second file to arrive with it is not "copy it and run it".
BUNDLE ?=
.PHONY: check test vet fmt build clean host
.PHONY: check test vet fmt build clean host hosts bootstrap packaging-test
check: fmt vet test packaging-test build
@@ -57,5 +57,44 @@ host:
exit $$status
@echo "built for $(SYSTEM) carrying $(BUNDLE)"
# The installer, carrying the control plane's image:
# make bootstrap IMAGE=mesh-control:v1.2.3
#
# The image is BUILT ELSEWHERE and handed over — mesh-control's own `make image` — and embedded
# here at release time. Not built on the machine being bootstrapped, and not fetched: the forge
# that holds mesh-control's source runs on the mesh, so a bootstrap that had to fetch or build the
# control plane would need a mesh in order to raise one. Carrying it breaks that cycle, the same
# way carrying the bundle breaks the "copy it onto a machine and run it" one (novox/hq ADR 0005).
#
# The saved image occupies the embed slot for the length of one build and the placeholder goes
# back, exactly as `host:` does with the bundle. Nothing large is ever committed.
#
# IMAGE must be a NAME:TAG and not an id. The installer identifies the carried image by its tag,
# because an image id is the digest of the image's configuration and a runtime REWRITES that
# configuration as it loads — so the id in the archive is not the id the receiving machine will
# hold, and the tag is the only name that survives the transfer. Saving by id produces an archive
# with no tags at all, which the installer refuses; caught here instead, in front of the person who
# can fix it.
#
# BOOTSTRAP_OUT is where the binary is written, and it exists because something other than a person
# now builds this: the lab rebuilds every artifact it runs from source before a raise, into paths it
# chose (mesh-lab's src/rebuild.ts, and novox/hq 04-ISSUES/005 for why it does that at all). A
# caller that could not say where the output goes would have to copy it afterwards, which is one
# more step to forget.
BOOTSTRAP_OUT ?= mesh-bootstrap
bootstrap:
@test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; }
@case "$(IMAGE)" in sha256:*) echo "IMAGE=$(IMAGE) is an image id. The installer identifies the carried image by its tag, because an id is the digest of a configuration that a runtime rewrites as it loads. Pass a name:tag"; exit 1;; esac
@docker image inspect "$(IMAGE)" >/dev/null 2>&1 || { echo "this machine does not hold $(IMAGE) — build it in mesh-control with 'make image'"; exit 1; }
@test -n "$$(docker image inspect --format '{{len .RepoTags}}' "$(IMAGE)" | grep -v '^0$$')" || { echo "$(IMAGE) has no repository tag, so the saved archive would carry no name the installer can ask a runtime about. Tag it first: docker tag $(IMAGE) mesh-control:<version>"; exit 1; }
@cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder
@docker save --output internal/image/control-plane.tar "$(IMAGE)"
@CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o "$(BOOTSTRAP_OUT)" ./cmd/mesh-bootstrap; \
status=$$?; \
mv internal/image/control-plane.tar.placeholder internal/image/control-plane.tar; \
exit $$status
@echo "built $(BOOTSTRAP_OUT) carrying $(IMAGE)"
clean:
rm -f mesh-host
rm -f mesh-host mesh-bootstrap
cmd/mesh-bootstrap/main.go
+283
View File
@@ -0,0 +1,283 @@
// Command mesh-bootstrap brings a mesh into existence on a bare machine.
//
// Tier 0, beside `mesh-host` and not inside it. Bootstrapping is done by hand and it changes a
// machine, which is what tier 0 is (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) — but
// `mesh-host` states of itself that it connects to nothing and listens on nothing and that what it
// applies comes from a file, and that is the whole reason an always-running root daemon can be
// audited by reading one page. An installer that loads images and interrogates a control plane
// cannot be folded into it without making that sentence false. Same tier, same repository,
// different program.
//
// Genesis is a pivot (novox/hq ADR 0067). It raises a substrate whose control plane is named by the
// digest of its own configuration — legal exactly where nothing could have served an image — then
// enrols this machine, installs the registry module, pushes that image into it to get the manifest
// digest it has never had, reinstalls the control plane as an ordinary module pinned to it, and
// drops the temporary one. Without --catalog it stops after the substrate and says why.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/bootstrap"
"github.com/novox/mesh-host/internal/store"
)
// version is stamped at build time. Unset in a development build, and said so rather than
// defaulted to something that looks like a release.
var version = "development build"
const (
defaultTemplate = "substrate.lock"
defaultOut = "/var/lib/mesh-host/substrate.lock"
defaultRegistry = "127.0.0.1:5000"
defaultHost = "/usr/local/bin/mesh-host"
defaultService = "mesh-host.service"
)
const usage = `mesh-bootstrap — make a bare machine into a mesh
bootstrap the ten steps below (the default)
version
1 preflight what has to be true before anything is changed
2 load the control plane's image, carried in this installer
3 bundle the substrate, named for this machine
4 apply raise it
5 verify it is up, and the control plane replies
6 enrol this machine becomes the mesh's first node
7 registry install the module that gives this mesh an image store
8 publish push the control plane's image into it, for its first digest
9 control reinstall the control plane as an ordinary module, pinned to that digest
10 retire drop the temporary control plane; the host removes it
--bundle the substrate template to build this machine's bundle from
(default ` + defaultTemplate + `)
--out where the produced bundle is written, for a person to read
(default ` + defaultOut + `)
--state where this node records what it has applied
(default ` + store.DefaultPath + `)
--catalog a checkout of the mesh's catalogue, holding the registry's and the
control plane's manifests. Without it this stops after step 5
--node the name this machine is known by (default: its hostname)
--registry where this mesh keeps its own images (default ` + defaultRegistry + `)
every node pulls the control plane from this, so on a mesh of more
than one machine it must be an address the others can reach
--host the mesh-host binary on this machine (default ` + defaultHost + `)
--host-service the unit that supervises it (default ` + defaultService + `)
--host-in-background start the host unsupervised instead. It does not survive
a reboot. This is what a lab does and what no real machine should
--system which operating system this is; by default it is asked
--timeout how long any single probe may take (default 30s)
--wait how long a thing that is merely starting is given (default 3m)
--dry-run everything that does not change the machine
--json machine-readable output
Genesis is a pivot: a temporary control plane installs the registry that makes it
permanent. The temporary one is called temp-mesh-control and the permanent one is
called mesh-control, so they are two containers with two owners and there is nothing
to hand over.
Every step is idempotent: run it again after fixing whatever it named, and the steps
that already succeeded say so.
`
func main() {
// Ctrl-C must stop the installer, not be swallowed by whatever it is waiting for — and it
// waits on pulls, on a runtime starting, and on a control plane opening its stores.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
command, opts, jsonOut, err := parseArgs(os.Args[1:])
if err == nil {
err = run(ctx, command, opts, jsonOut)
}
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-bootstrap: %v\n", err)
os.Exit(1)
}
}
// parseArgs takes an optional subcommand first, then its flags.
//
// Parsed in a loop for the reason `mesh-host` records: the standard library stops at the FIRST
// non-flag argument, so a flag sitting after one is silently dropped and the command exits zero
// having ignored what it was asked. That fault has been paid for twice in this repository and is
// not being paid for a third time.
func parseArgs(args []string) (string, bootstrap.Options, bool, error) {
opts := bootstrap.Options{
Template: defaultTemplate,
Out: defaultOut,
State: store.DefaultPath,
Registry: defaultRegistry,
Host: defaultHost,
// The machine's own name, because that is what a person already calls it and an installer
// inventing a different one would leave the mesh naming a machine nobody recognises. It is
// read here rather than inside the bootstrap so that --node overrides a fact rather than a
// default computed halfway through.
Node: hostname(),
HostService: defaultService,
// Longer than the host's 10s: these probes reach a container runtime that may be busy
// pulling, and a probe that times out on a working machine is a false refusal.
Timeout: 30 * time.Second,
// A socket-activated runtime queued behind the network, and a control plane running its
// first `initdb`-shaped wait, are both minutes rather than seconds.
Wait: 3 * time.Minute,
}
var jsonOut bool
command := "bootstrap"
if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' {
command = args[0]
args = args[1:]
}
set := newFlagSet(&opts, &jsonOut)
var positionals []string
rest := args
for {
if err := set.Parse(rest); err != nil {
return "", opts, false, err
}
rest = set.Args()
if len(rest) == 0 {
break
}
positionals = append(positionals, rest[0])
rest = rest[1:]
}
// Refused rather than ignored: a mistyped argument that changes nothing and reports success is
// worse than an error, and this program's whole job is to change a machine.
if len(positionals) > 0 {
return "", opts, false, fmt.Errorf(
"unexpected argument %q — try `mesh-bootstrap help`", positionals[0])
}
return command, opts, jsonOut, nil
}
func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError)
set.SetOutput(os.Stderr)
set.Usage = func() { fmt.Fprint(os.Stderr, usage) }
set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from")
set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written")
set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied")
set.StringVar(&opts.Catalogue, "catalog", opts.Catalogue,
"a checkout of the mesh's catalogue; without it this stops after the substrate")
set.StringVar(&opts.Node, "node", opts.Node, "the name this machine is known by")
set.StringVar(&opts.Registry, "registry", opts.Registry, "where this mesh keeps its own images")
set.StringVar(&opts.Host, "host", opts.Host, "the mesh-host binary on this machine")
set.StringVar(&opts.HostService, "host-service", opts.HostService, "the unit that supervises it")
set.BoolVar(&opts.HostInBackground, "host-in-background", false,
"start the host unsupervised; it does not survive a reboot")
set.StringVar(&opts.System, "system", opts.System, "which operating system this is")
set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take")
set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given")
set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine")
set.BoolVar(jsonOut, "json", false, "machine-readable output")
return set
}
func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error {
switch command {
case "bootstrap":
say := func(line string) {
if !jsonOut {
fmt.Println(line)
}
}
result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{
Run: apply.ExecRunner,
Dial: dial,
Fetch: fetch,
}, say)
// Printed whichever way it went. What the installer got through before it stopped is on
// the machine either way, and a report that only exists on success describes a machine
// nobody has (novox/hq ADR 0018).
if jsonOut {
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if encodeErr := encoder.Encode(result); encodeErr != nil && err == nil {
return encodeErr
}
}
return err
case "version":
fmt.Println(version)
return nil
case "help", "-h", "--help":
fmt.Fprint(os.Stderr, usage)
return nil
default:
return fmt.Errorf("unknown command %q — try `mesh-bootstrap help`", command)
}
}
// hostname is what this machine calls itself, or empty.
//
// Empty rather than a guess: a machine that cannot say its own name is one the installer must be
// told about, and `mesh-bootstrap-0` would be a name in the mesh's records that matches nothing
// anybody types anywhere else. The refusal happens at step 6, where the name is first needed.
func hostname() string {
name, err := os.Hostname()
if err != nil {
return ""
}
return name
}
// fetch asks an HTTP endpoint and reports what it said.
//
// Plain HTTP, and only at the mesh's own registry: it is reached over the mesh's private network,
// which is already the encrypted and authenticated thing, and a second layer inside it would be
// certificates to issue and rotate for no property the first does not have (mesh-control's
// `internal/builder` pushes to it on the same reasoning).
//
// The body is read with a limit. What is asked for is a status and a short JSON answer, and a
// registry that answered with a gigabyte would otherwise be an installer that never returns.
func fetch(ctx context.Context, url string) (int, string, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return 0, "", err
}
response, err := http.DefaultClient.Do(request)
if err != nil {
return 0, "", err
}
defer response.Body.Close()
said, err := io.ReadAll(io.LimitReader(response.Body, 1<<20))
if err != nil {
return response.StatusCode, "", err
}
return response.StatusCode, string(said), nil
}
// dial answers whether a TCP address responds.
//
// A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a
// connection to exactly this address. A machine whose DNS resolves and whose route is missing
// passes a lookup and fails the thing that matters.
func dial(ctx context.Context, address string) error {
var dialer net.Dialer
conn, err := dialer.DialContext(ctx, "tcp", address)
if err != nil {
return err
}
return conn.Close()
}
cmd/mesh-bootstrap/main_test.go
+164
View File
@@ -0,0 +1,164 @@
package main
import (
"flag"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/bootstrap"
"github.com/novox/mesh-host/internal/store"
)
// Argument handling gets tests for the reason `mesh-host` records: the standard library stops
// parsing at the first non-flag argument, so a flag sitting after one is silently dropped and the
// command exits zero having ignored what it was asked. Here that would mean `--dry-run` ignored on
// a program whose whole job is to change a machine.
func TestBootstrapIsWhatItDoesWithNoCommand(t *testing.T) {
// Running the installer with nothing but flags must install, not print usage: the command is
// the reason the binary exists, and making somebody type its name twice buys nothing.
command, opts, _, err := parseArgs([]string{"--dry-run"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if command != "bootstrap" {
t.Errorf("command = %q, want bootstrap", command)
}
if !opts.DryRun {
t.Error("--dry-run before any subcommand was ignored")
}
}
func TestFlagsAreReadWhereverTheySit(t *testing.T) {
for _, args := range [][]string{
{"bootstrap", "--dry-run", "--bundle", "s.lock", "--json"},
{"bootstrap", "--json", "--bundle=s.lock", "--dry-run"},
{"--bundle", "s.lock", "--dry-run", "--json"},
} {
_, opts, jsonOut, err := parseArgs(args)
if err != nil {
t.Errorf("%v: unexpected error: %v", args, err)
continue
}
if !opts.DryRun || !jsonOut || opts.Template != "s.lock" {
t.Errorf("%v parsed as dry-run=%v json=%v bundle=%q",
args, opts.DryRun, jsonOut, opts.Template)
}
}
}
func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) {
// Asymmetric cost: an error is a moment's annoyance, and a silently dropped --dry-run is a
// machine changed by somebody who asked for it not to be.
if _, _, _, err := parseArgs([]string{"bootstrap", "--dry-runn"}); err == nil {
t.Fatal("a mistyped flag was accepted")
}
}
func TestAnUnexpectedArgumentIsRefused(t *testing.T) {
if _, _, _, err := parseArgs([]string{"bootstrap", "substrate.lock"}); err == nil {
t.Fatal("a stray argument was ignored rather than refused — the bundle is --bundle")
}
}
func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) {
// The usage text is a promise. A default that drifts from what is printed is a small lie that
// costs somebody an afternoon in front of a machine that will not come up.
_, opts, jsonOut, err := parseArgs(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if opts.Template != defaultTemplate {
t.Errorf("default bundle is %q; the usage text says %q", opts.Template, defaultTemplate)
}
if opts.Out != defaultOut {
t.Errorf("default out is %q; the usage text says %q", opts.Out, defaultOut)
}
if opts.State != store.DefaultPath {
t.Errorf("default state is %q; the host's own default is %q", opts.State, store.DefaultPath)
}
if opts.Timeout != 30*time.Second {
t.Errorf("default timeout is %s; the usage text says 30s", opts.Timeout)
}
if opts.Wait != 3*time.Minute {
t.Errorf("default wait is %s; the usage text says 3m", opts.Wait)
}
if opts.DryRun || jsonOut || opts.System != "" {
t.Error("something is on by default that the usage text describes as a flag")
}
}
// Every flag the usage text promises must exist, and every flag that exists must be in the usage
// text. The two drifting apart is how a program acquires a feature nobody can find and a
// documented option that does nothing.
func TestTheUsageTextAndTheFlagsAgree(t *testing.T) {
var opts bootstrap.Options
var jsonOut bool
set := newFlagSet(&opts, &jsonOut)
declared := map[string]bool{}
set.VisitAll(func(f *flag.Flag) { declared[f.Name] = true })
for name := range declared {
if !strings.Contains(usage, "--"+name) {
t.Errorf("--%s exists and the usage text does not mention it", name)
}
}
for _, promised := range []string{
"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json",
"catalog", "node", "registry", "host", "host-service", "host-in-background",
} {
if !declared[promised] {
t.Errorf("the usage text promises --%s and no such flag exists", promised)
}
}
}
// The pivot's defaults are the documented ones too, and the one that has no default is the one
// that decides whether the pivot happens at all.
func TestThePivotsDefaultsAreTheDocumentedOnes(t *testing.T) {
_, opts, _, err := parseArgs(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if opts.Registry != defaultRegistry {
t.Errorf("default registry is %q; the usage text says %q", opts.Registry, defaultRegistry)
}
if opts.Host != defaultHost {
t.Errorf("default host binary is %q; the usage text says %q", opts.Host, defaultHost)
}
if opts.HostService != defaultService {
t.Errorf("default host service is %q; the usage text says %q",
opts.HostService, defaultService)
}
if opts.HostInBackground {
t.Error("the host is started unsupervised by default, and no real machine should")
}
// **No default, deliberately.** A catalogue this installer went looking for on its own would
// be a checkout somebody else made, at whatever commit they left it on — and it decides which
// image the mesh's control plane is pinned to for ever after.
if opts.Catalogue != "" {
t.Errorf("--catalog defaults to %q; without one the installer stops at the substrate",
opts.Catalogue)
}
// The machine's own name, because that is what a person already calls it.
if opts.Node == "" {
t.Error("no default node name; this machine can say what it is called")
}
}
// --node overrides the machine's own name rather than being ignored because a default was already
// computed. The name is what the mesh's records, tokens, assignments and pushes all name.
func TestTheNodeNameCanBeSaid(t *testing.T) {
_, opts, _, err := parseArgs([]string{"--node", "anchor", "--catalog", "/somewhere"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if opts.Node != "anchor" {
t.Errorf("--node anchor parsed as %q", opts.Node)
}
if opts.Catalogue != "/somewhere" {
t.Errorf("--catalog parsed as %q", opts.Catalogue)
}
}
internal/apply/apply.go
+9
View File
@@ -1154,6 +1154,15 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error {
if _, err := run(ctx, cri, "image", "inspect", image); err == nil {
return nil
}
// An image named by the digest of its own configuration is one this machine was supposed to
// already hold — built here, or handed over. There is no registry that answers for it, so
// pulling would fail somewhere that names a network problem instead of a missing image.
if strings.HasPrefix(image, "sha256:") {
return fmt.Errorf(
"image %s is not on this machine, and an image named by its own digest cannot be "+
"fetched: nothing serves it. Build it here, or load it, before applying this",
image)
}
if _, err := run(ctx, cri, "pull", image); err != nil {
return fmt.Errorf("pulling image %s: %w", image, err)
}
internal/bootstrap/apply.go
+134
View File
@@ -0,0 +1,134 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"strings"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// Runner is the same runner every applier in this repository takes.
type Runner = apply.Runner
// ApplyBundle raises the substrate, through the host's own apply.
//
// **This calls `internal/apply` rather than running the `mesh-host` binary**, and that is worth
// stating because shelling out would have been easier. The installer and the host must apply a
// declaration identically — same removal pass, same read-backs, same refusal model, same record of
// what this machine now owns — and two code paths that must behave the same are two code paths
// that will not. The `mesh-host` binary is also not guaranteed to be on a machine this program is
// raising, which would make the installer depend on the thing it installs.
//
// It applies under `store.OriginCarried`, which is the same origin `mesh-host reconcile` uses and
// is not a detail: what the substrate raised must be invisible to the removal pass of a
// declaration that later arrives from the control plane, or the first thing the mesh tells this
// node would tear down the mesh (novox/hq 04-ISSUES/010).
//
// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version,
// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is
// not one.
func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration,
run Runner, say func(string)) (apply.Report, error) {
// Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host`
// does: finding out half way through is the half-configured machine tier 0 exists to prevent.
if err := system.Check(sys, d); err != nil {
return apply.Report{}, err
}
known, err := store.Load(o.State)
if err != nil {
return apply.Report{}, err
}
report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run,
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed)
// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a
// failure is on the machine either way, and a host that did not record it would believe it
// owns less than it does and leave that behind for ever.
if saveErr := store.Save(o.State, updated); saveErr != nil {
if applyErr != nil {
return report, fmt.Errorf("%w\n\nand this node's state could not be saved: %v",
applyErr, saveErr)
}
return report, saveErr
}
if applyErr != nil {
return report, fmt.Errorf("%w\n\nThe machine is in whatever state that left it. Fix what "+
"is named above and run this again — every step is idempotent, and the ones that "+
"already succeeded will say so", applyErr)
}
return report, nil
}
// refuseSealed is what happens when a bundle contains a file the mesh sealed to this node.
//
// It cannot happen and it is refused with a sentence rather than a nil dereference. A sealing key
// is generated at enrolment (`internal/identity`), and enrolment is something that happens on a
// mesh — which is the thing this program is raising. A substrate bundle carrying a sealed file
// would be a bundle written for a node that has already joined.
func refuseSealed(string) ([]byte, error) {
return nil, errors.New(
"this bundle contains a file sealed to a node's key, and a machine that has not enrolled " +
"has no such key. A substrate is applied before any mesh exists, so it can carry no " +
"secret the mesh sealed")
}
// WorkOutSystem decides which half of the host applies things on this machine, and proves it.
//
// `mesh-host` pins this at link time because it is built for one operating system and refuses to
// touch a machine without knowing which (novox/hq ADR 0005). An installer run by hand has no
// link-time to pin it at, so it asks — but it does not guess: every system already knows how to
// prove it is the one it claims to be, by asking its package database about a package that is
// certainly there. Exactly one may answer.
//
// A machine where none answers is refused with what each of them said, because "unsupported
// system" is a sentence nobody can act on and "pacman does not answer here" is.
func WorkOutSystem(ctx context.Context, run Runner, named string) (system.System, error) {
if strings.TrimSpace(named) != "" {
chosen, err := system.For(named)
if err != nil {
return nil, err
}
if err := chosen.Confirm(ctx, run); err != nil {
return nil, fmt.Errorf("--system %s was given, and this machine says otherwise: %w",
named, err)
}
return chosen, nil
}
var answered []system.System
var refusals []string
for _, candidate := range system.All() {
if err := candidate.Confirm(ctx, run); err != nil {
refusals = append(refusals, fmt.Sprintf(" %s: %v", candidate.Name(), err))
continue
}
answered = append(answered, candidate)
}
switch len(answered) {
case 1:
return answered[0], nil
case 0:
return nil, fmt.Errorf(
"this machine is none of the systems this installer knows how to change, so nothing "+
"was attempted:\n%s\nName one with --system if it is really one of them and its "+
"package database is merely unwell", strings.Join(refusals, "\n"))
default:
var names []string
for _, s := range answered {
names = append(names, s.Name())
}
return nil, fmt.Errorf(
"this machine answers as %s at once, and the installer must not choose between them: "+
"package names and unit names differ, and picking wrong misconfigures the machine "+
"quietly. Say which with --system", strings.Join(names, " and "))
}
}
internal/bootstrap/apply_test.go
+91
View File
@@ -0,0 +1,91 @@
package bootstrap
import (
"context"
"errors"
"strings"
"testing"
)
// `mesh-host` is built for one operating system and pins it at link time. An installer run by hand
// has no link time, so it asks — and it does not guess: each system already knows how to prove it
// is the one it claims to be, by asking its package database about a package that is certainly
// there. Getting this wrong installs with the wrong package manager and the wrong unit names.
func TestTheMachineIsAskedWhichSystemItIs(t *testing.T) {
// Only pacman answers, so this is the arch host and nothing had to be told so.
onlyPacman := func(_ context.Context, name string, _ ...string) (string, error) {
if name == "pacman" {
return "pacman 7.0.0-1\n", nil
}
return "", errors.New("command not found")
}
chosen, err := WorkOutSystem(context.Background(), onlyPacman, "")
if err != nil {
t.Fatal(err)
}
if chosen.Name() != "arch" {
t.Errorf("this machine was worked out to be %q", chosen.Name())
}
}
// A machine that is none of them is refused with what each of them said. "Unsupported system" is
// a sentence nobody can act on; "pacman does not answer here" is.
func TestAMachineThatIsNoneOfThemIsRefusedWithWhatEachSaid(t *testing.T) {
nothing := func(context.Context, string, ...string) (string, error) {
return "", errors.New("command not found")
}
_, err := WorkOutSystem(context.Background(), nothing, "")
if err == nil {
t.Fatal("a machine that answers as no known system was accepted")
}
for _, wanted := range []string{"arch:", "alpine:", "--system"} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
}
}
}
// And a machine that was TOLD what it is still has to prove it. Installing the arch half of the
// host on Alpine must say so once, at the start, rather than failing later inside pacman.
func TestASystemThatWasNamedIsStillProved(t *testing.T) {
onlyApk := func(_ context.Context, name string, _ ...string) (string, error) {
if name == "apk" {
return "apk-tools-2.14.0\n", nil
}
return "", errors.New("command not found")
}
if _, err := WorkOutSystem(context.Background(), onlyApk, "arch"); err == nil {
t.Fatal("--system arch was believed on a machine where pacman does not answer")
}
if _, err := WorkOutSystem(context.Background(), onlyApk, "alpine"); err != nil {
t.Errorf("--system alpine was refused on a machine where apk answers: %v", err)
}
}
func TestASystemNobodyHasBuiltIsRefusedByName(t *testing.T) {
anything := func(context.Context, string, ...string) (string, error) { return "", nil }
_, err := WorkOutSystem(context.Background(), anything, "debian")
if err == nil {
t.Fatal("--system debian was accepted, and no debian host is built")
}
if !strings.Contains(err.Error(), "arch") {
t.Errorf("the refusal does not say which systems exist: %v", err)
}
}
// A substrate is applied before any mesh exists, so it can carry no secret the mesh sealed — there
// is no key to open one with. Refused with a sentence rather than a nil dereference.
func TestASealedFileInASubstrateIsRefusedWithAReason(t *testing.T) {
_, err := refuseSealed("anything")
if err == nil {
t.Fatal("a sealed file in a substrate bundle was accepted")
}
if !strings.Contains(err.Error(), "has not enrolled") {
t.Errorf("the refusal does not say why there is no key: %v", err)
}
}
internal/bootstrap/bootstrap.go
+477
View File
@@ -0,0 +1,477 @@
// Package bootstrap brings a mesh into existence on a bare machine.
//
// **Why this is a program at all.** Until now the only complete written-down copy of the
// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is
// why every gap in it kept being found late and by accident: an install procedure that lives as a
// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is
// that procedure, made into the thing it always was.
//
// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by
// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is
// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and
// listens on nothing and that what it applies comes from a file — a property that is what makes an
// always-running root daemon auditable, and that must stay literally true. This program pulls
// images and asks a running control plane questions. Same tier, same repository, different binary.
//
// **No registry is required for the mesh's own image, and no source either.** The control plane
// exists in no registry by design, and the forge that holds its source runs on the mesh — so a
// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer
// carries the image (`internal/image`) and names it by its image id: the sha256 of its own
// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq
// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream
// `name@sha256:` references and are pulled from the internet like anything else.
//
// **And that id is read back from the machine, never predicted from the archive.** The digest of a
// configuration is not portable: a runtime rewrites the configuration as it loads, so the same
// bytes are held under a different name on the machine that receives them than on the one that
// saved them. The archive is identified by its TAG, which does survive the transfer, and the id
// the bundle names is whatever the runtime answers for that tag afterwards. See Load, which
// carries the measurement.
package bootstrap
import (
"context"
"fmt"
"strings"
"time"
)
// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
// nobody can act on and this will be run over and over by somebody getting a machine working.
type Step string
const (
StepPreflight Step = "preflight"
StepLoad Step = "load"
StepBundle Step = "bundle"
StepApply Step = "apply"
StepVerify Step = "verify"
StepEnrol Step = "enrol"
StepRegistry Step = "registry"
StepPublish Step = "publish"
StepControlPlane Step = "control-plane"
StepRetire Step = "retire"
)
// Steps in the order they happen, so a failure can say "step 2 of 10".
//
// The first five make a machine; the last five make a mesh that can maintain itself. They are one
// program because they are one procedure — the whole reason the pivot exists is that steps 7 to 9
// cannot happen without steps 1 to 5, and steps 1 to 5 leave something that cannot be upgraded
// without steps 7 to 9 (novox/hq ADR 0067).
var Steps = []Step{
StepPreflight, StepLoad, StepBundle, StepApply, StepVerify,
StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire,
}
// Error is a failure, named by the step it happened in.
type Error struct {
Step Step
Err error
}
func (e *Error) Error() string {
at := 0
for i, s := range Steps {
if s == e.Step {
at = i + 1
}
}
return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err)
}
func (e *Error) Unwrap() error { return e.Err }
func failed(step Step, err error) error {
if err == nil {
return nil
}
return &Error{Step: step, Err: err}
}
// Options are the things that differ between machines.
type Options struct {
// Template is the substrate bundle this machine's own bundle is made from.
Template string
// Out is where the produced bundle is written, so a person can read what was applied.
Out string
// State is where the host records what it has applied here — the same file `mesh-host` reads,
// because what this raises the host must afterwards own.
State string
// System is which half of the host applies things. Empty means ask the machine.
System string
// DryRun does everything that does not change the machine.
DryRun bool
// Timeout bounds any single probe.
Timeout time.Duration
// Wait is how long something that is merely starting is given: a socket-activated container
// runtime, a control plane opening its stores.
Wait time.Duration
// Node is the name this machine is known by in the mesh. Everything after the substrate names
// it: the record, the token, the assignment, the push.
Node string
// Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and
// the control plane's manifests are read from. Empty stops the installer after the substrate:
// there is no pivot without manifests, and pretending otherwise would leave a machine that
// looks installed and cannot upgrade itself.
Catalogue string
// Registry is where this mesh's own images live, as this machine reaches it. Every node will
// pull the control plane from what this says, so on a mesh of more than one machine it must be
// an address the others can reach.
Registry string
// Host is the `mesh-host` binary on this machine — the program that enrols and then holds the
// machine to what the mesh says. The installer runs it; it does not contain it.
Host string
// HostService is the unit that supervises it. Started and enabled, never written: what a unit
// says is a packaging decision, and an installer inventing one would put a file on the machine
// that whatever installed the host will disagree with.
HostService string
// HostInBackground starts the host unsupervised instead, which is what the lab does and what no
// real machine should do — it does not survive a reboot.
HostInBackground bool
}
// pivots reports whether this run goes past the substrate.
func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" }
// Deps are the ways this program reaches outside itself. Injected so the whole of it can be
// tested without a container runtime, a network, or a machine to break — the same reason
// `internal/apply` takes a Runner (novox/hq ADR 0017).
type Deps struct {
// Run executes a command. apply.ExecRunner in production.
Run Runner
// Dial reports whether a TCP address answers, for "can this machine reach the registries the
// bundle names".
Dial func(ctx context.Context, address string) error
// Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own
// registry: a container that is up is not a registry that serves, and `/v2/` is the one
// question whose answer means it is.
Fetch func(ctx context.Context, url string) (int, string, error)
}
// Result is what the bootstrap did, in the shape `--json` prints.
type Result struct {
System string `json:"system"`
DryRun bool `json:"dry-run,omitempty"`
// Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and
// what the produced bundle names it by.
Image string `json:"image,omitempty"`
// ImageArchive is what the carried archive calls the same image. Reported because it is
// routinely a DIFFERENT id: a runtime rewrites an image's configuration as it loads, and an id
// is that configuration's digest. Never what the bundle names.
ImageArchive string `json:"image-in-archive,omitempty"`
// ImageTag is the name the runtime was asked by, which is how the id above was obtained.
ImageTag string `json:"image-tag,omitempty"`
// ImageTags is everything the image was called when it was saved.
ImageTags []string `json:"image-tags,omitempty"`
// ImageHeld is true when the machine already held it and nothing was loaded.
ImageHeld bool `json:"image-already-held,omitempty"`
// ImagePredicted is true when Image is the archive's id because nothing was loaded — a dry run
// only, and the reason a dry run does not claim to know what would be applied.
ImagePredicted bool `json:"image-id-is-a-prediction,omitempty"`
// Bundle is where the produced bundle was written, and what was done to produce it.
Bundle string `json:"bundle,omitempty"`
BundleWas string `json:"bundle-replaced,omitempty"`
BundlePlaces int `json:"bundle-places,omitempty"`
BundleWrote bool `json:"bundle-written,omitempty"`
// Applied is how many resources the apply reported on, and whether any of them moved.
Applied int `json:"applied,omitempty"`
Changed bool `json:"changed,omitempty"`
// Running is the substrate's containers, confirmed up.
Running []string `json:"running,omitempty"`
// Answered is what the temporary control plane said back — not merely that it is up.
Answered string `json:"temporary-control-plane,omitempty"`
// Temporary is what the substrate's control plane is called, which is not what the module's is.
Temporary string `json:"temporary-container,omitempty"`
// Node is this machine's name in the mesh, and how it came to be enrolled and heard from.
Node string `json:"node,omitempty"`
NodeAdded bool `json:"node-record-created,omitempty"`
Enrolled bool `json:"enrolled-now,omitempty"`
Agent string `json:"host-agent,omitempty"`
// Registry is the mesh's own artifact store, once it answers.
Registry string `json:"registry,omitempty"`
RegistryReplied int `json:"registry-replied,omitempty"`
RegistryKnown bool `json:"registry-already-registered,omitempty"`
RegistryRunning string `json:"registry-container,omitempty"`
PublishedAs string `json:"control-plane-image,omitempty"`
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
// Permanent is the control plane as an ordinary module.
Permanent string `json:"permanent-container,omitempty"`
PermanentAnswered string `json:"permanent-control-plane,omitempty"`
StoresDelivered []string `json:"stores-delivered,omitempty"`
TemporaryRetired bool `json:"temporary-retired,omitempty"`
TemporaryWasGone bool `json:"temporary-was-already-gone,omitempty"`
TemporaryRemovedAt int `json:"resources-removed,omitempty"`
// Stopped names why a run went no further. Empty on a run that pivoted.
Stopped string `json:"stopped,omitempty"`
}
// Run performs the bootstrap, saying what it is doing as it goes.
//
// Every step is idempotent, and every step says whether it found something or changed it. That is
// not politeness: this program is run repeatedly while somebody gets a machine working, and a step
// that cannot tell "already done" from "just done" makes the second run indistinguishable from the
// first — which is how a person stops believing any of it.
//
// It does not retry. A pull that failed for a reason that goes away by itself is real, and the
// answer to it is to run this again: re-running is the retry, and it is one a person chooses after
// reading which step failed and why.
//
// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is
// named by the digest of its own configuration, because nothing has ever served that image and
// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine
// enrols, the registry module is installed, the carried image is pushed INTO that registry — which
// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary
// module pinned to it. The temporary one is then dropped from the bundle and the host removes it.
//
// **What makes the last part expressible is a name.** The substrate's control plane is called
// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners:
// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction
// by omission is the right end for something named "temp".
//
// Without --catalog it stops after step 5 and says so, because there are no manifests to install
// and a machine that looks installed and cannot upgrade itself is worse than one that stopped.
//
// **What an interruption leaves, at every step, and how a re-run continues.** This matters more
// here than anywhere else in the repository, because a machine left without a control plane cannot
// be fixed remotely — so no step may leave one:
//
// 1–3 nothing on the machine but a written file. Re-run: the bundle is produced again.
// 4 a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest.
// 5 everything up; something did not answer yet. Re-run: it is asked again.
// 6 a node record and possibly a spent token. Re-run: `node list` finds the record, the
// identity file says whether this machine enrolled, and a fresh token is issued if not.
// 7 the registry registered, assigned, maybe not applied. Re-run: registered again (an
// upsert), pushed again, waited for again. The temporary control plane is untouched.
// 8 the image pushed and the digest unread. Re-run: the registry is asked what it holds and
// the answer is the same digest; nothing is pushed twice.
// 9 the module registered and the container not yet up, OR up beside the temporary one. Both
// are working states: the mesh has a control plane throughout. Re-run: continues.
// 10 the bundle rewritten and the container still there. Re-run: the bundle already omits it
// and the apply removes it; a bundle that already omits it reports "already dropped".
//
// The only step that cannot be undone by re-running is enrolment, and that is refused rather than
// repeated: a second identity is one the mesh does not know, and the mesh believes the first.
func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) {
if say == nil {
say = func(string) {}
}
result := Result{DryRun: o.DryRun}
// ---- 1. preflight -------------------------------------------------------------------
say("preflight — what has to be true before anything is changed")
template, err := Preflight(ctx, o, d, say)
if err != nil {
return result, failed(StepPreflight, err)
}
// Which half of the host applies things here. Asked of the machine and proved, because
// `mesh-host` pins this at link time and an installer run by hand has no link time.
sys, err := WorkOutSystem(ctx, d.Run, o.System)
if err != nil {
return result, failed(StepPreflight, err)
}
result.System = sys.Name()
say(" system " + sys.Name())
// ---- 2. load ------------------------------------------------------------------------
say("load — the control plane's image, carried in this installer")
loaded, err := Load(ctx, d.Run, o.DryRun, say)
if err != nil {
return result, failed(StepLoad, err)
}
result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held
result.ImageArchive, result.ImageTag = loaded.Archive, loaded.Tag
result.ImagePredicted = loaded.Predicted
// ---- 3. bundle ----------------------------------------------------------------------
say("bundle — what this machine will be asked to be")
rewritten, err := Rewrite(template, loaded.ID)
if err != nil {
return result, failed(StepBundle, err)
}
result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out
result.Temporary = rewritten.TempName
if rewritten.Renamed {
say(fmt.Sprintf(" control plane %s, renamed from %s",
rewritten.TempName, rewritten.WasCalled))
say(" the permanent one is a module and takes the plain name; " +
"this one is dropped at the end")
} else {
say(" control plane " + rewritten.TempName + " — the template already named it that")
}
if rewritten.Changed {
say(fmt.Sprintf(" its image %s", rewritten.Now))
say(fmt.Sprintf(" replacing %s, named in %d place(s)",
rewritten.Was, rewritten.Places))
} else {
say(fmt.Sprintf(" its image %s — the template already named it, nothing rewritten",
rewritten.Now))
}
if loaded.Predicted {
say(" UNCONFIRMED that id is the archive's and no runtime has been asked. A real " +
"run reads it back.")
}
for _, kept := range rewritten.Kept {
say(" left alone " + kept)
}
if rewritten.BrokerAddress != "" {
// Said every time, and never changed. Every enrolment token this mesh issues will tell a
// joining node to dial this address, and a wrong one is silent until the second node fails
// to come back. The installer does not know this machine's address and will not invent it.
say(" nodes will dial " + rewritten.BrokerAddress +
" — check this is an address other machines can reach")
}
if o.DryRun {
// Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it
// if wrong while changing nothing. The bundle has been produced and re-parsed in memory,
// which is everything that could be checked without touching the machine; what is left is
// loading, applying and asking the result questions, and none of those can be answered by
// not doing them.
say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources))
if o.pivots() {
// Named rather than attempted. Everything from step 6 on is a conversation with a
// control plane that a dry run has not raised, so there is nothing to ask and nothing
// honest to report about the answers.
say(" would then enrol " + o.Node + ", install the registry from " +
o.Catalogue + ", push the control plane's image into it,")
say(" reinstall the control plane as a module, and drop " +
rewritten.TempName)
}
result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " +
"loaded or applied"
if loaded.Predicted {
result.Stopped += ". The image id in it is the archive's own and is not necessarily " +
"the one this machine would hold — a runtime rewrites an image's configuration as " +
"it loads, and the id is that configuration's digest"
}
say("\n" + result.Stopped)
return result, nil
}
// **Nothing predicted is ever written down.** The line above is the only path on which
// `loaded.ID` can be the archive's id, and it returns. Asserted here rather than left to the
// reader, because what would follow is a bundle naming an image this machine does not hold —
// and nothing serves an image named by the digest of its own configuration, so it would fail
// inside a pull that cannot succeed, three steps from the cause.
if loaded.Predicted {
return result, failed(StepBundle, fmt.Errorf(
"the control plane's image id was never confirmed against this machine's runtime, and "+
"the bundle was about to be written with it. This is a fault in the installer, not "+
"in the machine"))
}
if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil {
return result, failed(StepBundle, err)
}
result.BundleWrote = true
say(fmt.Sprintf(" wrote %s (%d resources) — read it, this is what is applied",
o.Out, rewritten.Resources))
// ---- 4. apply -----------------------------------------------------------------------
say("apply — raising the substrate")
report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say)
result.Applied, result.Changed = len(report.Outcomes), report.Changed()
if err != nil {
return result, failed(StepApply, err)
}
if report.Changed() {
say(fmt.Sprintf(" applied %d resource(s)", len(report.Outcomes)))
} else {
say(fmt.Sprintf(" already matches %d resource(s) checked, nothing moved",
len(report.Outcomes)))
}
// ---- 5. verify ----------------------------------------------------------------------
say("verify — the substrate is up, and the control plane replies")
verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say)
result.Running, result.Answered = verified.Running, verified.Answered
if err != nil {
return result, failed(StepVerify, err)
}
if !o.pivots() {
// Stopped, and said plainly. What has been raised works and cannot be upgraded: its
// control plane is named by an image id, which no registry serves, so nothing can ever
// replace it with a newer one. That is the whole of what the pivot fixes, and it needs
// manifests, and manifests come from a checkout somebody has to point this at.
result.Stopped = "no --catalog was given, so this stopped at the substrate. " +
"The control plane is named by the digest of its own configuration and no registry " +
"serves it, so this mesh cannot yet upgrade itself. Run again with " +
"--catalog <a checkout of the mesh's catalogue> to finish the pivot; every step " +
"above will say it is already done"
say("\nthis machine is a mesh of one node, with nothing joined to it yet.")
say(result.Stopped)
return result, nil
}
// ---- 6. enrol -------------------------------------------------------------------------
//
// From here on the mesh is being told things, and the way to tell it anything is to run its
// own binary inside its own container. `temporary` is the substrate's control plane; the
// module's is a different container with a different name and does not exist yet.
temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout}
say("enrol — this machine joins the mesh it is running")
enrolled, err := Enrol(ctx, o, sys, temporary, say)
result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined
result.Agent = enrolled.Agent
if err != nil {
return result, failed(StepEnrol, err)
}
// ---- 7. registry ----------------------------------------------------------------------
say("registry — somewhere for this mesh to keep its own images")
registry, err := InstallRegistry(ctx, o, d, temporary, say)
result.Registry, result.RegistryRunning = registry.Address, registry.Container
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
if err != nil {
return result, failed(StepRegistry, err)
}
// ---- 8. publish -----------------------------------------------------------------------
say("publish — the control plane's image gets its first manifest digest")
published, err := PublishControlPlane(ctx, o, d, loaded.ID, say)
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
if err != nil {
return result, failed(StepPublish, err)
}
// ---- 9. control plane -----------------------------------------------------------------
say("control plane — installed as an ordinary module, pinned to that digest")
permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration,
published.Reference, say)
result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered
result.StoresDelivered = permanent.Delivered
if err != nil {
return result, failed(StepControlPlane, err)
}
// ---- 10. retire -----------------------------------------------------------------------
say("retire — the temporary control plane is dropped from the bundle")
retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say)
result.TemporaryRetired = retired.Gone || retired.Already
result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed
if err != nil {
return result, failed(StepRetire, err)
}
say("\nthis machine is a mesh of one node, and the control plane it runs is a module " +
"pinned to an image its own registry serves.")
say("what remains is somebody else's: adding nodes, and assigning what they should run.")
return result, nil
}
internal/bootstrap/control.go
+410
View File
@@ -0,0 +1,410 @@
package bootstrap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-host/internal/declaration"
)
// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the
// environment.
//
// `MESH_STORE_<CONTEXT>` is what the control plane reads (mesh-control's `internal/store`.Variable)
// and putting a password in a container's environment puts it in `docker inspect` for ever. So a
// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value
// into that file on the machine, and nothing but the process reads it.
const storeFileSuffix = "_FILE"
// storeVariablePrefix is the front of the same names.
const storeVariablePrefix = "MESH_STORE_"
// Permanent is what step 9 did.
type Permanent struct {
Installed
// Container is what the module calls its container, confirmed running.
Container string
// Image is what it is pinned to — the digest step 8's push produced.
Image string
// Delivered is every store connection accepted into it, by secret name.
Delivered []string
// Answered is what the permanent control plane said back.
Answered string
}
// InstallControlPlane makes the control plane an ordinary module.
//
// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary
// control plane composes a declaration naming the registry-pinned image, publishes it, and this
// node's host creates the container. Nothing is asked to replace itself while running, which is
// what makes the whole thing expressible: the container being created is called `mesh-control` and
// the one composing it is called `temp-mesh-control`, so there are two of them and neither is in
// the other's way.
//
// **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.**
// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are
// the credentials the substrate bundle created the databases with. Generating replacements would
// put thirty-two random bytes where a working connection string has to be, and the control plane
// would come up unable to open a single context. So they go in through `secret accept`, which is
// exactly the path for a value the mesh must carry and could not have invented — and they are read
// out of the bundle this installer produced rather than reconstructed, because the bundle is what
// created them and a second opinion about what a DSN should say is a second chance to be wrong.
func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane,
substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) {
out := Permanent{Image: image}
manifest, err := readManifest(o.Catalogue, ControlPlaneModule)
if err != nil {
return out, fmt.Errorf(
"%w\n"+
"This is the manifest that makes the control plane an ordinary module. Without it "+
"the machine keeps the temporary control plane the substrate raised, which works "+
"and cannot be upgraded — so the install stops here rather than pretending to "+
"have pivoted", err)
}
pinned, places, err := pinImage(manifest, image)
if err != nil {
return out, err
}
say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places))
container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned))
if err != nil {
return out, err
}
out.Container = container
if container == "" {
return out, fmt.Errorf(
"the %s module's container has no name, so nothing can be verified afterwards",
ControlPlaneModule)
}
installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say)
out.Installed = installed
if err != nil {
return out, err
}
// The connections, before the push that would otherwise deliver random bytes for them.
delivered, err := deliverStores(ctx, o, control, pinned, substrate, say)
out.Delivered = delivered
if err != nil {
return out, err
}
if out.Pushed, err = pushNode(ctx, o, control, say); err != nil {
return out, err
}
if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil {
return out, err
}
// And it answers, which is the same question step 5 asked of the temporary one and for the
// same reason: `status` opens all three stores, so a reply proves the sealed connections it
// was given are the ones the substrate made. Asked of the NEW container — this is the only
// moment in the program where two control planes are running, and asking the wrong one would
// report the temporary one's health as the permanent one's.
answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say)
if err != nil {
return out, err
}
out.Answered = answered
return out, nil
}
// pinImage replaces the catalogue's placeholder digest with what the registry assigned.
//
// **Textual, and every place it appears.** A manifest may name its image in more than one resource
// — the catalogue's converted modules routinely carry a runtime container beside the application's
// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves
// something pointing at an image nothing serves, and it fails half way through an apply rather
// than here.
//
// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already
// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a
// control plane that is not the image this machine just published — which is the one thing this
// step exists to guarantee.
func pinImage(manifest []byte, reference string) ([]byte, int, error) {
places := bytes.Count(manifest, []byte(placeholderDigest))
if places == 0 {
return nil, 0, fmt.Errorf(
"the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+
"pin to the image this machine just published.\n"+
"A manifest already naming a digest was pinned by somebody else, to some other "+
"build. Registering it would install a control plane that is not the one this "+
"installer carried and pushed", ControlPlaneModule, placeholderDigest)
}
// The reference the registry gave back is `<registry>/<repository>@sha256:…`, and what the
// manifest holds is `<something>@sha256:0…0`. Replacing only the digest would leave the
// manifest's own repository name in front of it — which may be `mesh-control` with no
// registry, and a runtime would then pull it from the internet. The whole reference moves.
var out bytes.Buffer
rest := manifest
for {
at := bytes.Index(rest, []byte(placeholderDigest))
if at < 0 {
out.Write(rest)
break
}
// Back up over the repository this digest belongs to, which runs to the opening quote.
start := bytes.LastIndexByte(rest[:at], '"')
if start < 0 {
return nil, 0, fmt.Errorf(
"the %s module's manifest has a placeholder digest that is not inside a JSON "+
"string, so the installer cannot tell what image it belongs to", ControlPlaneModule)
}
out.Write(rest[:start+1])
out.WriteString(reference)
rest = rest[at+len(placeholderDigest):]
}
pinned := out.Bytes()
// Read back. A substitution on text can catch more than it was aimed at, and the manifest is
// about to be handed to the mesh as the description of what it runs.
var checked map[string]any
if err := json.Unmarshal(pinned, &checked); err != nil {
return nil, 0, fmt.Errorf(
"pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err)
}
if bytes.Contains(pinned, []byte(placeholderDigest)) {
return nil, 0, fmt.Errorf(
"the %s module's manifest still carries a placeholder digest after pinning",
ControlPlaneModule)
}
return pinned, places, nil
}
// controlPlaneResourceIn is the id of the resource that runs the control plane.
//
// The manifest is written by the catalogue and the installer does not get to name its resources.
// What it can do is find the one container whose image is the one just pinned — and when a manifest
// declares exactly one container, that is the answer without any searching at all.
func controlPlaneResourceIn(manifest []byte) string {
var m struct {
Resources []struct {
ID string `json:"id"`
Type string `json:"type"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return ""
}
var containers []string
for _, r := range m.Resources {
if r.Type == "container" {
containers = append(containers, r.ID)
}
}
if len(containers) == 1 {
return containers[0]
}
// More than one, so the name has to be guessed at rather than derived — and the catalogue's
// own convention for the resource that IS the module is `container`, with anything else beside
// it named for what it does.
for _, id := range containers {
if id == "container" || id == ControlPlaneModule || id == "control-plane" {
return id
}
}
return ""
}
// deliverStores carries the substrate's own database connections into the module.
//
// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls
// these secrets is what is delivered. The installer does not guess that the secret holding the
// inventory connection is called `inventory`; it follows the manifest from the variable to the
// secret, and a manifest whose two ends do not meet is refused rather than half-delivered.
//
// **What is delivered is what the substrate already has, and only that.** The mesh generates an
// own-secret nobody supplied, which is right for something coming into existence and wrong for
// something that already exists. So every variable the module fills from a secret is looked up in
// the substrate's control plane: what it names is accepted, what it does not is left for the mesh
// to make. A store connection missing from the substrate is the one exception and is an error —
// a control plane that cannot open a context is not a control plane.
func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte,
substrate *declaration.Declaration, say func(string)) ([]string, error) {
wanted, err := secretsByVariableIn(manifest)
if err != nil {
return nil, err
}
if !anyStoreIn(wanted) {
return nil, fmt.Errorf(
"the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+
"each context through its own credential (novox/hq ADR 0008), so a manifest naming "+
"none describes a control plane that can open nothing.\n"+
"The shape this installer delivers into is a file per context, named by an "+
"own-secret, with %s<CONTEXT>%s pointing at it — directly, or at where that file is "+
"mounted inside the container",
ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix)
}
temporary, err := controlPlaneIn(substrate)
if err != nil {
return nil, err
}
var delivered []string
for _, variable := range sortedKeys(wanted) {
secret := wanted[variable]
value := strings.TrimSpace(temporary.Env[variable])
if value == "" {
if strings.HasPrefix(variable, storeVariablePrefix) {
return delivered, fmt.Errorf(
"the %s module wants %s and the bundle this installer produced does not name "+
"one.\n"+
"That connection is the substrate's, created at genesis — the mesh cannot "+
"invent it and the installer will not guess at one",
ControlPlaneModule, variable)
}
// Not something the substrate made. The mesh generates its own, which is exactly what
// an own-secret is for; said so that nothing about the delivery is silent.
say(" the mesh will make " + secret + " — the substrate names no " + variable)
continue
}
// Into the container as a file, because `secret accept` reads a file or a prompt and the
// installer has neither a terminal to be prompted at nor a way to write to a command's
// standard input through the runner every applier in this repository shares.
at := "/accepting-" + secret
if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
return delivered, err
}
if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret,
"--from", at); err != nil {
return delivered, err
}
delivered = append(delivered, secret)
say(" accepted " + secret + " — " + variable + ", as the substrate made it")
}
return delivered, nil
}
// secretsByVariableIn maps each environment variable the module fills from a secret to that
// secret's name.
//
// Two shapes, because the catalogue uses both:
//
// - `MESH_STORE_<CONTEXT>_FILE` in the container's environment, naming a path the process reads.
// The path may be the own-secret's own path, or — more usually — where that file is mounted
// inside the container, in which case the volumes say which is which. Following the mount is
// not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at
// `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and
// refuse a correct manifest.
// - `VAR=${secret:name}` inside a file resource the container reads its environment from, which
// is how a value that is not a path gets in at all.
//
// A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and
// the process would find an empty file where a credential has to be, which presents as a container
// that will not start, a long way from the cause.
func secretsByVariableIn(manifest []byte) (map[string]string, error) {
var m struct {
OwnSecrets map[string]string `json:"own-secrets"`
Resources []struct {
Type string `json:"type"`
Path string `json:"path"`
Content string `json:"content"`
Env map[string]string `json:"env"`
Volumes []string `json:"volumes"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
}
secretAt := map[string]string{}
for name, path := range m.OwnSecrets {
secretAt[path] = name
}
wanted := map[string]string{}
for _, r := range m.Resources {
switch r.Type {
case "file":
for variable, secret := range secretsInContent(r.Content) {
wanted[variable] = secret
}
case "container":
inside := mountedFrom(r.Volumes)
for key, path := range r.Env {
if !strings.HasPrefix(key, storeVariablePrefix) ||
!strings.HasSuffix(key, storeFileSuffix) {
continue
}
on := path
if from, mounted := inside[path]; mounted {
on = from
}
secret, named := secretAt[on]
if !named {
return nil, fmt.Errorf(
"the %s module's container reads %s from %s, and no own-secret of that "+
"module writes that file.\n"+
"So the mesh would seal nothing there and the control plane would find "+
"an empty file where a connection string has to be. The manifest has to "+
"name the two ends the same, directly or through a mount",
ControlPlaneModule, key, path)
}
wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret
}
}
}
return wanted, nil
}
// mountedFrom is where each path inside a container comes from outside it.
func mountedFrom(volumes []string) map[string]string {
inside := map[string]string{}
for _, volume := range volumes {
parts := strings.Split(volume, ":")
if len(parts) < 2 {
continue
}
inside[parts[1]] = parts[0]
}
return inside
}
// secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment
// from.
func secretsInContent(content string) map[string]string {
found := map[string]string{}
for _, line := range strings.Split(content, "\n") {
variable, value, is := strings.Cut(strings.TrimSpace(line), "=")
if !is {
continue
}
const opens = "${secret:"
if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") {
continue
}
found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}")
}
return found
}
// anyStoreIn reports whether any of these variables is a context's connection.
func anyStoreIn(wanted map[string]string) bool {
for variable := range wanted {
if strings.HasPrefix(variable, storeVariablePrefix) {
return true
}
}
return false
}
func sortedKeys(m map[string]string) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
internal/bootstrap/control_test.go
+275
View File
@@ -0,0 +1,275 @@
package bootstrap
import (
"context"
"strings"
"testing"
"time"
)
// Step 9 is where the control plane stops being a special case. These tests defend the two things
// that could go wrong quietly: pinning it to the wrong image, and delivering it store connections
// the mesh invented rather than the ones the substrate actually made.
// theControlPlaneModule is the catalogue's manifest, trimmed to what this installer reads.
//
// A fixture rather than the file itself, unlike the substrate example the rewrite tests use: the
// catalogue is a different repository on a different branch, and a test that read it would pass or
// fail according to what somebody else had checked out. What it must stay faithful to is the
// SHAPE — the placeholder digest, the own-secret per context, the mount from the machine's path to
// the container's, and the environment file that fills what is not a path.
const theControlPlaneModule = `{
"module": "mesh-control",
"version": "1",
"slug": "control",
"capabilities": ["container-runtime"],
"claims": [{"name": "the-control-plane", "scope": "mesh"}],
"own-secrets": {
"inventory": "/var/lib/mesh/mesh-control/inventory",
"identity": "/var/lib/mesh/mesh-control/identity",
"licences": "/var/lib/mesh/mesh-control/licences",
"broker": "/var/lib/mesh/mesh-control/broker",
"broker-management": "/var/lib/mesh/mesh-control/broker-management"
},
"resources": [
{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},
{"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-control/broker.env",
"mode": "0600",
"content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
{"id": "server", "type": "container", "name": "mesh-control",
"image": "mesh-control@` + placeholderDigest + `",
"network": "host", "args": ["serve"],
"env-file": ["/var/lib/mesh/mesh-control/broker.env"],
"env": {
"MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
"MESH_STORE_IDENTITY_FILE": "/run/secrets/identity",
"MESH_STORE_LICENCES_FILE": "/run/secrets/licences",
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
},
"volumes": [
"mesh-broker-tls:/broker-tls:ro",
"/var/lib/mesh/mesh-control/inventory:/run/secrets/inventory:ro",
"/var/lib/mesh/mesh-control/identity:/run/secrets/identity:ro",
"/var/lib/mesh/mesh-control/licences:/run/secrets/licences:ro"
]}
]
}`
const pushedReference = "127.0.0.1:5000/mesh-control@sha256:" +
"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
// **The whole reference moves, not only the digest.** The manifest's placeholder names a
// repository too, and replacing sixty-four zeros inside it would leave `mesh-control@sha256:…`
// with no registry in front — which a runtime would go to the internet for, and this mesh's
// control plane exists in no public registry by design.
func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) {
pinned, places, err := pinImage([]byte(theControlPlaneModule), pushedReference)
if err != nil {
t.Fatal(err)
}
if places != 1 {
t.Errorf("the placeholder was found in %d place(s)", places)
}
if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) {
t.Errorf("the manifest does not name the pushed image:\n%s", pinned)
}
if strings.Contains(string(pinned), `"mesh-control@sha256:`) {
t.Errorf("the digest was replaced and the manifest's own repository name was left in "+
"front of it, so nothing says which registry serves it:\n%s", pinned)
}
}
// A manifest already naming a real digest was pinned by somebody else, to some other build.
// Registering it would install a control plane that is not the image this machine just published,
// which is the one thing this step exists to guarantee.
func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) {
already := strings.Replace(theControlPlaneModule, placeholderDigest,
"sha256:"+strings.Repeat("9", 64), 1)
if _, _, err := pinImage([]byte(already), pushedReference); err == nil {
t.Fatal("a manifest already pinned to some other image was accepted")
}
}
// Every placeholder moves. A manifest naming its image in a second resource — a runtime container
// beside the application's, which the catalogue's converted modules routinely carry — would
// otherwise be left half pinned, and fail inside an apply rather than here.
func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) {
twice := strings.Replace(theControlPlaneModule,
`{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},`,
`{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},
{"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true,
"image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1)
pinned, places, err := pinImage([]byte(twice), pushedReference)
if err != nil {
t.Fatal(err)
}
if places != 2 {
t.Errorf("the placeholder was found in %d place(s), and the manifest names it twice", places)
}
if strings.Contains(string(pinned), placeholderDigest) {
t.Error("a placeholder survived the pinning")
}
}
// **The connections are the substrate's, and they are read out of the bundle that made them.**
// The mesh cannot invent them: they are the credentials the substrate created the databases with,
// and thirty-two random bytes in their place would leave the control plane unable to open a single
// context. The pairing is read from the manifest so that whatever the catalogue calls these
// secrets is what is delivered.
func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
wanted, err := secretsByVariableIn([]byte(theControlPlaneModule))
if err != nil {
t.Fatal(err)
}
// **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads
// them at /run/secrets. Matching on the path alone would find nothing and refuse a correct
// manifest, which is exactly the ordinary case in the catalogue.
for variable, secret := range map[string]string{
"MESH_STORE_INVENTORY": "inventory",
"MESH_STORE_IDENTITY": "identity",
"MESH_STORE_LICENCES": "licences",
"MESH_BROKER_AMQP": "broker",
"MESH_BROKER_MANAGEMENT": "broker-management",
} {
if wanted[variable] != secret {
t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret)
}
}
// And what the manifest fills from the machine rather than from a secret is left alone.
if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed {
t.Error("the address the mesh composes from the machine was treated as a secret")
}
// The values are the substrate's own, taken from the produced bundle rather than composed.
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}
delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control,
[]byte(theControlPlaneModule), rewritten.Declaration, func(string) {})
if err != nil {
t.Fatal(err)
}
// Three stores and both halves of the broker: everything the substrate made and nothing else.
if len(delivered) != 5 {
t.Fatalf("%d values were delivered, and the substrate names five: %v",
len(delivered), delivered)
}
for _, secret := range delivered {
if !runtime.ran("secret accept anchor mesh-control " + secret + " --from") {
t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands)
}
}
}
// A secret the substrate did not make is left for the mesh to make, and said so. Every other
// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh.
func TestASecretTheSubstrateNeverMadeIsLeftToTheMesh(t *testing.T) {
extra := strings.Replace(theControlPlaneModule,
`"broker": "/var/lib/mesh/mesh-control/broker",`,
`"broker": "/var/lib/mesh/mesh-control/broker",
"something-new": "/var/lib/mesh/mesh-control/something-new",`, 1)
extra = strings.Replace(extra,
`"content": "MESH_BROKER_AMQP=${secret:broker}\n`,
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1)
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}
var said []string
delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control,
[]byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) })
if err != nil {
t.Fatal(err)
}
for _, secret := range delivered {
if secret == "something-new" {
t.Error("a value the substrate never made was accepted as though it had")
}
}
if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") {
t.Errorf("nothing was said about the secret the mesh has to make: %v", said)
}
}
// A manifest whose container reads a file no own-secret writes is refused. The mesh would seal
// nothing there and the control plane would find an empty file where a connection string has to
// be — which presents as a control plane that will not start, three steps from the cause.
func TestAConnectionFileNothingWritesIsRefused(t *testing.T) {
mismatched := strings.Replace(theControlPlaneModule,
`"inventory": "/var/lib/mesh/mesh-control/inventory",`,
`"inventory": "/var/lib/mesh/mesh-control/somewhere-else",`, 1)
_, err := secretsByVariableIn([]byte(mismatched))
if err == nil {
t.Fatal("a manifest whose two ends do not meet was accepted")
}
if !strings.Contains(err.Error(), "own-secret") {
t.Errorf("the refusal does not say which half is missing: %v", err)
}
}
// A manifest asking for no store connections at all describes a control plane that can open
// nothing, and the refusal says what shape the installer delivers into — because the manifest is
// written in another repository and this is where the two have to agree.
func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) {
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}
bare := `{"module":"mesh-control","version":"1","resources":[
{"id":"container","type":"container","name":"mesh-control",
"image":"mesh-control@` + placeholderDigest + `"}]}`
_, err = deliverStores(context.Background(), Options{Node: "anchor"}, control,
[]byte(bare), rewritten.Declaration, func(string) {})
if err == nil {
t.Fatal("a control plane that can open nothing was accepted")
}
if !strings.Contains(err.Error(), storeVariablePrefix+"<CONTEXT>"+storeFileSuffix) {
t.Errorf("the refusal does not say what shape is expected: %v", err)
}
}
// The permanent control plane is asked a question, not merely looked at — the same question the
// temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so
// a reply proves the sealed connections it was given are the ones the substrate made.
func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) {
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
"module list": "",
"exec mesh-control /mesh-control": "1 node, 0 waiting\n",
})}
control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
out, err := InstallControlPlane(context.Background(),
installing(t, catalogueWith(t, ControlPlaneModule, theControlPlaneModule)),
Deps{Run: runtime.run}, control, rewritten.Declaration, pushedReference, func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Answered != "1 node, 0 waiting" {
t.Errorf("the permanent control plane's reply is reported as %q", out.Answered)
}
if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") {
t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands)
}
// And the module was registered with the digest, not with the placeholder.
if !runtime.ran("module add /mesh-control-module.json") {
t.Errorf("the module was never registered: %v", runtime.commands)
}
}
internal/bootstrap/enrol.go
+316
View File
@@ -0,0 +1,316 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/system"
)
// hereIs what `node list` says about a machine the mesh has heard from recently.
//
// mesh-control prints one of three words per node: "here", "never spoken", or "out of touch <age>".
// The installer waits for the first, and it is the only honest proof that the host agent is
// running: an enrolled machine whose host is not running looks exactly like an enrolled machine
// whose host has crashed, and both look exactly like a successful install until the first push
// silently applies nothing.
const hereIs = "here"
// Enrolled is what step 6 did.
type Enrolled struct {
// Node is the name this machine is known by.
Node string
// Added is true when the mesh had no record and one was created.
Added bool
// Joined is true when this machine enrolled during this run. False on a re-run.
Joined bool
// Agent says how the host came to be running: what was found, or what was started.
Agent string
}
// Enrol makes this machine the mesh's first node, and gets the host agent running on it.
//
// **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a
// control plane that answers — a mesh of one node in the sense that it has one machine and zero
// node records. Everything after this point is the control plane being *told* things, and none of
// it reaches a machine until a host is running there to hear it.
//
// Four things, in this order, each asked before it is done:
//
// 1. a node record, unless `node list` already shows one
// 2. a one-time token, unless this machine already holds an identity
// 3. `mesh-host enrol`, which is the machine presenting the identity it already had
// 4. the host agent running, and the mesh saying it has heard from it
//
// **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an
// identity the mesh has recorded, and enrolling again would replace it with a second one the mesh
// does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer
// earlier and with a sentence about what to do.
//
// `control.run` is this machine's runner and not only the control plane's: the same injected
// Runner reaches the container, the service manager and the host binary, which is what lets the
// whole of this be tested without any of the three.
func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane,
say func(string)) (Enrolled, error) {
out := Enrolled{Node: o.Node}
if strings.TrimSpace(o.Node) == "" {
return out, errors.New(
"this machine has no name to be known by. Give one with --node; it is what the mesh " +
"records, what a token is issued against, and what every later assignment names")
}
// 1. The record.
nodes, err := control.tell(ctx, "node", "list")
if err != nil {
return out, err
}
if mentions(nodes, o.Node) {
say(" already a node " + o.Node)
} else {
if _, err := control.tell(ctx, "node", "add", o.Node); err != nil {
return out, err
}
out.Added = true
say(" node record " + o.Node)
}
// 2 and 3. The identity, and being known.
//
// Asked of this machine's own identity file rather than of the mesh, because the two answer
// different questions: the mesh knows whether a record exists, and only the machine knows
// whether it holds the key that record names.
where := identity.Path(o.State)
switch mine, err := identity.Load(where); {
case err == nil && mine.Node == o.Node:
say(" already enrolled " + o.Node + " — " + where)
case err == nil:
return out, fmt.Errorf(
"this machine is already node %q and was asked to become %q.\n"+
"Re-enrolling replaces the identity the mesh has recorded, so it is not something "+
"an installer does on its own. Run with --node %s, or remove %s and start over "+
"deliberately", mine.Node, o.Node, mine.Node, where)
case !errors.Is(err, identity.ErrNoIdentity):
return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err)
default:
said, err := control.tell(ctx, "token", "issue", "--node", o.Node)
if err != nil {
return out, err
}
token, err := tokenIn(said)
if err != nil {
return out, err
}
joining, cancel := context.WithTimeout(ctx, o.Wait)
joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State)
cancel()
if err != nil {
return out, fmt.Errorf(
"%s would not enrol this machine: %w\n%s\n"+
"The token is one-time and has now been spent; running this again issues "+
"another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined)))
}
if !strings.Contains(joined, "enrolled as "+o.Node) {
// Exit zero and no such sentence. Refused rather than believed: the host says exactly
// this line on success, and something that succeeded without saying it did something
// else.
return out, fmt.Errorf(
"%s exited happily and did not say it enrolled as %s:\n%s",
o.Host, o.Node, indent(strings.TrimSpace(joined)))
}
out.Joined = true
say(" enrolled as " + o.Node)
}
// 4. The agent.
agent, err := runTheHost(ctx, o, sys, control, say)
if err != nil {
return out, err
}
out.Agent = agent
return out, nil
}
// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
//
// **The installer does not install the service, and says so.** A unit file is a packaging decision
// — where the binary lives, which user it runs as, what it is called — and an installer that
// invented one would be putting a file on the machine that whatever installed `mesh-host` will
// later disagree with. So this starts a unit that is already there and refuses when there is none.
//
// It goes through the host's OWN system abstraction rather than running systemctl itself, for the
// reason `ApplyBundle` gives about applying: two implementations of "is this service running" is
// how the installer and the host come to disagree about a machine. It also gets the right refusal
// for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped",
// which is exactly the distinction that matters here.
//
// --host-in-background is the lab's arrangement, kept because the lab is what exercises this and
// it has no service. It is loud about what it is, because a host started this way is gone at the
// next reboot and the mesh would go quiet for reasons nobody would connect to an install.
func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane,
say func(string)) (string, error) {
// **Asked of both, because either one alone lies.**
//
// A process in the table may be wedged and never collect anything, which is why this asked the
// mesh instead. But the mesh having heard from a node proves only that something spoke to it
// *once*, and `mesh-host enrol` — run moments earlier, by this very step — is itself that
// something. So straight after enrolling, the mesh has always heard from this machine and no
// agent is running; skipping the start on that signal alone is exactly wrong.
//
// What it costs is silent and total. Every later step is the control plane being *told*
// things, and nothing it is told reaches a machine with no agent to collect it: the push at
// step 7 is accepted, the mesh records the module, and no container is ever created. It
// surfaces three minutes later as "the registry is not there at all" — one step from its
// cause, looking nothing like it.
if heard, err := heardFrom(ctx, control, o.Node); err != nil {
return "", err
} else if heard {
running, err := agentIsRunning(ctx, o, sys, control)
if err != nil {
return "", err
}
if running {
say(" host running the mesh has heard from " + o.Node)
return "already running", nil
}
say(" host running the mesh has heard from " + o.Node + ", and no agent is running " +
"here — enrolling speaks once, which is not the same thing")
}
how := ""
switch {
case o.HostInBackground:
// Detached, and not waited for: this process runs until the machine stops, so a runner
// that captures output would never return. `nohup … &` through a shell is how the lab
// starts it and is deliberately the same command.
started, cancel := context.WithTimeout(ctx, o.Timeout)
_, err := control.run(started, "sh", "-c",
fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State))
cancel()
if err != nil {
return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err)
}
how = "started in the background"
say(" host running started in the background — THIS DOES NOT SURVIVE A REBOOT.")
say(" A real machine needs " + o.HostService + " installed and enabled.")
default:
state, err := sys.ServiceState(ctx, control.run, o.HostService)
if err != nil {
return "", fmt.Errorf(
"the mesh has not heard from %s and this machine has no %s to start: %w\n"+
"The host has to be running for anything the mesh says to reach this machine. "+
"Install the service that supervises it and run this again — every step "+
"before this one will say it is already done. In a lab, --host-in-background "+
"starts it unsupervised instead, and that is not an install",
o.Node, o.HostService, err)
}
if state != "running" {
if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil {
return "", fmt.Errorf("cannot start %s: %w", o.HostService, err)
}
how = "started " + o.HostService
say(" host running started " + o.HostService)
} else {
// Running, and the mesh has not heard from it. Not an error yet — it may have started
// a second ago — so it is waited for below like everything else that is merely
// starting.
how = o.HostService + " was already running"
say(" host running " + o.HostService + " is running")
}
// At boot as well, or the machine comes back without a mesh and nothing says why.
if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" {
if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil {
return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err)
}
say(" host at boot " + o.HostService + " enabled")
}
}
// Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node
// are two different facts, and only the second is the one every later step rests on.
deadline := time.Now().Add(o.Wait)
for {
heard, err := heardFrom(ctx, control, o.Node)
if err != nil {
return "", err
}
if heard {
say(" the mesh hears " + o.Node)
return how, nil
}
if time.Now().After(deadline) {
return "", fmt.Errorf(
"%s is running and the mesh has not heard from %s after %s.\n"+
"The host links to the broker at the address the token carried — if that "+
"address is not one this machine can reach, this is where it shows. Read the "+
"host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+
"installer wrote", o.HostService, o.Node, o.Wait)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(answerEvery):
}
}
}
// agentIsRunning is whether a host agent is on this machine now — the other half of the question
// the mesh cannot answer.
//
// Deliberately not an error when there is nothing to find: "no unit here" and "not running" are
// both simply *not running* to this caller, and the branch that starts one says far more about a
// missing unit than this could, naming what to install.
func agentIsRunning(ctx context.Context, o Options, sys system.System, control controlPlane) (bool, error) {
if o.HostInBackground {
// No unit to ask, so the process table is all there is. The exit status is the whole
// answer; anything it printed is not this function's business.
if _, err := control.run(ctx, "pgrep", "-f", o.Host+" run"); err != nil {
return false, nil
}
return true, nil
}
state, err := sys.ServiceState(ctx, control.run, o.HostService)
if err != nil {
return false, nil
}
return state == "running", nil
}
// heardFrom asks the mesh whether this node has spoken to it.
func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) {
listing, err := control.tell(ctx, "node", "list")
if err != nil {
return false, err
}
for _, line := range strings.Split(listing, "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[0] == node {
return fields[1] == hereIs, nil
}
}
return false, nil
}
// tokenIn finds the token in what `token issue` said.
//
// The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation
// around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies
// it without this having to know the format — which is what keeps the installer from having an
// opinion about a thing the control plane owns.
func tokenIn(said string) (string, error) {
for _, line := range strings.Split(said, "\n") {
line = strings.TrimSpace(line)
if len(line) > 100 && !strings.ContainsAny(line, " \t") {
return line, nil
}
}
return "", fmt.Errorf(
"the mesh issued a token and there is no token in what it said:\n%s",
indent(strings.TrimSpace(said)))
}
internal/bootstrap/enrol_test.go
+263
View File
@@ -0,0 +1,263 @@
package bootstrap
import (
"context"
"crypto/ed25519"
"fmt"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/system"
)
// Step 6 is where the mesh stops being something running on a machine and starts being something
// the machine belongs to. What these tests defend is that it cannot happen twice, and that
// "installed" is never claimed for a machine the mesh has not actually heard from.
// alreadyEnrolled writes an identity file, as `mesh-host enrol` leaves behind.
func alreadyEnrolled(t *testing.T, node string) string {
t.Helper()
state := filepath.Join(t.TempDir(), "state.json")
mine, err := identity.Generate(node)
if err != nil {
t.Fatal(err)
}
// A whole membership, because an identity that cannot reach its mesh is refused on the way in
// — which is the right refusal and not the one being tested here.
signer, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
mine.Membership = identity.Membership{
Broker: "192.0.2.10:5671", Fingerprint: "sha256:whatever",
Signer: signer, Password: "issued-at-enrolment",
}
if err := identity.Save(identity.Path(state), mine); err != nil {
t.Fatal(err)
}
return state
}
func arch(t *testing.T) system.System {
t.Helper()
chosen, err := system.For("arch")
if err != nil {
t.Fatal(err)
}
return chosen
}
// A machine that has already enrolled is not enrolled again, and no token is spent on it. The
// installer is run over and over; a second identity is one the mesh does not know, and the mesh
// believes the first.
func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
runtime := &asked{answer: func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
return "anchor here 01J0\n", nil
// An agent is running here too. Both halves are needed: enrolling makes the mesh hear from
// a machine once, so the first answer alone also describes a machine with no agent at all.
case name == "pgrep":
return "4242\n", nil
}
return "", fmt.Errorf("unexpected: %s %v", name, args)
}}
out, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
}, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Joined || out.Added {
t.Error("a machine that had already enrolled enrolled again")
}
if runtime.ran("token issue") {
t.Errorf("a token was issued for a machine that already holds an identity: %v",
runtime.commands)
}
if out.Agent != "already running" {
t.Errorf("the host agent is reported as %q", out.Agent)
}
}
// The mesh having heard from a machine is not the same as an agent running on it, and enrolling is
// itself the thing that makes the mesh hear. Taken as proof of life it ends the step believing an
// agent it never started, and everything after is the control plane being told things that never
// reach the machine: the next push is accepted, recorded, and applied by nobody.
func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) {
runtime := &asked{answer: func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
// Exactly what enrolling leaves behind, with nothing running.
return "anchor here 01J0\n", nil
case name == "pgrep":
return "", fmt.Errorf("exit status 1")
case name == "sh":
return "", nil
}
return "", fmt.Errorf("unexpected: %s %v", name, args)
}}
out, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
}, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Agent == "already running" {
t.Fatal("a machine with no agent was reported as already running it")
}
if !runtime.ran("nohup") {
t.Errorf("no agent was started on a machine that has none: %v", runtime.commands)
}
}
// A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling
// replaces the identity the mesh recorded, which is a deliberate act and not something an
// installer does on its own.
func TestAMachineEnrolledUnderAnotherNameIsRefused(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if strings.Contains(strings.Join(args, " "), "node list") {
return "somewhere-else here 01J0\n", nil
}
return "", nil
}}
_, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "somewhere-else"), Timeout: time.Second,
}, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err == nil {
t.Fatal("a machine already enrolled as something else was enrolled again")
}
for _, wanted := range []string{"somewhere-else", "--node"} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
}
}
}
// **The mesh having heard from the node is the proof, not a process existing.** A host that is
// running and cannot reach the broker looks exactly like a successful install until the first push
// silently applies nothing — which is the class of fault this whole program exists to stop being
// found late.
func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
runtime := &asked{answer: func(_ string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
// Enrolled, and never spoken.
return "anchor never spoken 01J0\n", nil
case args[0] == "show":
return "LoadState=loaded\nActiveState=active\n", nil
case args[0] == "is-enabled":
return "enabled\n", nil
}
return "", nil
}}
_, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service",
Timeout: time.Second, Wait: 0,
}, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err == nil {
t.Fatal("a node the mesh has never heard from was reported enrolled and running")
}
if !strings.Contains(err.Error(), "MESH_BROKER_ADDRESS") {
t.Errorf("the failure does not name the thing that is silently fatal when wrong:\n%v", err)
}
}
// A machine with no service to start is refused, and the refusal says what is missing rather than
// inventing a unit file. What a unit says is a packaging decision, and an installer writing one
// would put a file on the machine that whatever installed the host will disagree with.
func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
return "anchor never spoken 01J0\n", nil
case args[0] == "show":
// systemd knows nothing about it.
return "LoadState=not-found\nActiveState=inactive\n", nil
}
return "", nil
}}
_, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service",
Timeout: time.Second, Wait: 0,
}, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err == nil {
t.Fatal("a machine with no host service was reported as having a running host")
}
for _, wanted := range []string{"mesh-host.service", "--host-in-background"} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
}
}
}
// A machine with no name is refused before anything is said to the mesh. The name is what the
// record, the token, the assignment and the push all name, and one the installer invented would
// match nothing anybody types anywhere else.
func TestAMachineWithNoNameIsRefusedBeforeAnythingIsAsked(t *testing.T) {
runtime := &asked{answer: func(string, []string) (string, error) {
return "", fmt.Errorf("nothing should have been asked")
}}
_, err := Enrol(context.Background(), Options{Timeout: time.Second}, arch(t),
controlPlane{container: "temp-mesh-control", run: runtime.run}, func(string) {})
if err == nil {
t.Fatal("a machine with no name was enrolled")
}
if len(runtime.commands) != 0 {
t.Errorf("the mesh was asked something first: %v", runtime.commands)
}
}
// The token is found in what the mesh said, by the rule the lab uses: the one long unbroken line.
// The installer does not parse a format the control plane owns.
func TestTheTokenIsFoundInWhatTheMeshSaid(t *testing.T) {
said := "a token for anchor, good once:\n\n " + strings.Repeat("t", 240) + "\n\n" +
"carry it to the machine and run: mesh-host enrol --token <token>\n"
token, err := tokenIn(said)
if err != nil {
t.Fatal(err)
}
if token != strings.Repeat("t", 240) {
t.Errorf("the token was read as %q", token)
}
if _, err := tokenIn("nothing here that looks like one\n"); err == nil {
t.Fatal("an answer with no token in it was accepted")
}
}
// A listing's name is matched as a whole word at the start of a line, so `registry` is not found
// inside `registry-mirror`. A substring match would report a module installed that is not, and the
// installer would skip creating it.
func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) {
listing := "registry-mirror 1 built abc\nother 1 built def\n"
if mentions(listing, "registry") {
t.Error("registry-mirror was read as registry")
}
if !mentions(listing, "registry-mirror") {
t.Error("registry-mirror was not found")
}
}
internal/bootstrap/load.go
+199
View File
@@ -0,0 +1,199 @@
package bootstrap
import (
"context"
"fmt"
"os"
"strings"
"github.com/novox/mesh-host/internal/image"
)
// Loaded is the control plane's image on this machine.
type Loaded struct {
// ID is what THIS RUNTIME holds the image as, read back from it after the load. It is what the
// bundle names, and outside a dry run it is never a prediction — see Load.
ID string
// Archive is what the carried tar calls the same image. Kept because the two differ in
// practice, and a report showing only one of them cannot say that they did. Never what the
// bundle names.
Archive string
// Tag is the name the runtime is asked by. Load-bearing rather than decoration: it is the one
// name that survives `docker save` and `docker load` unchanged.
Tag string
// Tags is everything the archive was called when it was saved.
Tags []string
// Held is true when the machine already held it and nothing moved.
Held bool
// Predicted is true only on a dry run, where nothing was loaded and ID is therefore the
// archive's id — which is not necessarily the one this machine would end up with.
Predicted bool
}
// Load puts the carried control-plane image into this machine's container runtime, and reports
// what the runtime decided to call it.
//
// **The digest of a configuration is not portable across runtimes, and that is why the id is read
// back rather than predicted.** An image id is the sha256 of the image's configuration document,
// and a runtime REWRITES that document as it loads: a newer Docker saves in one format, an older
// one stores it in another, and the same layers come out under a different name. Measured on a
// live raise, an image saved as `sha256:b86bb81c…` on a workstation was loaded as
// `sha256:2dc21904…` on the machine it was carried to.
//
// This code used to read the id out of the tar before the runtime was asked anything and use it
// for both idempotence and the bundle. That is right on the machine the image was built on and
// wrong on every machine it is carried to — which is every machine this program exists for. The
// bundle would have named an image the machine does not hold; nothing serves an image named by
// the digest of its own configuration, which is the whole point of naming one that way; and the
// apply would have stopped inside a pull that cannot succeed. The lab hit exactly this.
//
// **So the image is identified by its TAG.** A tag is ordinary metadata the tar carries through
// unchanged, and asking the runtime what a tag resolves to is asking the only party entitled to
// answer. The tag never reaches the bundle — a pinned bundle may not rely on one
// (novox/hq ADR 0006) — it is how the id is obtained, not what is written down.
//
// **Idempotence is decided from what the runtime holds.** The tag is asked before the load and
// again after: the same id either side means nothing moved, which is a fact about this machine
// rather than a guess about the file. A tag that already resolves means the image is already
// held, and nothing is loaded at all.
//
// It reads back (novox/hq ADR 0018). A load that reported success and left nothing there is a
// failure, not a convergence.
func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loaded, error) {
saved, err := image.Saved()
if err != nil {
return Loaded{}, err
}
return loadImage(ctx, run, saved, dryRun, say)
}
// loadImage is Load with the carried bytes handed in, so the whole path can be tested against a
// saved image a test builds rather than against whatever a particular build embedded.
func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say func(string)) (Loaded, error) {
archiveID, err := image.ArchiveID(saved)
if err != nil {
return Loaded{}, err
}
loaded := Loaded{Archive: archiveID, Tags: image.Tags(saved)}
// **An untagged archive is a build-time fault, refused here rather than worked around.**
// Without a tag there is no portable name to ask the runtime about, and the only thing left is
// scraping the sentence `docker load` prints for a person — which differs between runtime
// versions and is exactly the kind of guess this whole step exists to stop making. The release
// target tags the image; an installer built without one was built wrong.
loaded.Tag = firstOr(loaded.Tags, "")
if loaded.Tag == "" {
return loaded, fmt.Errorf(
"the carried control-plane image has no tag, so there is no portable name to ask this "+
"machine's runtime what id it gave it.\n"+
"An image id is the digest of the image's configuration and a runtime rewrites "+
"that as it loads, so the id in the archive (%s) is not necessarily the id this "+
"machine will hold — and a bundle naming the wrong one names an image nothing "+
"serves. Rebuild the installer with a tagged image: `make bootstrap "+
"IMAGE=<name>:<tag>`", archiveID)
}
// Already held? Asked of the runtime, by the tag, before anything is written anywhere.
before, err := idOfImage(ctx, run, loaded.Tag)
if err != nil {
return loaded, err
}
if before != "" {
loaded.ID, loaded.Held = before, true
say(" already held " + before + " as " + loaded.Tag + " — nothing loaded")
sayIfDifferent(say, archiveID, before)
return loaded, nil
}
if dryRun {
// Nothing is loaded, so the runtime has not been asked to decide anything — and what it
// would decide cannot be worked out from here. Said, rather than quietly guessed at.
loaded.ID, loaded.Predicted = archiveID, true
say(fmt.Sprintf(" would load %s (%d bytes) as %s", archiveID, len(saved), loaded.Tag))
say(" NOT THE FINAL ID a runtime rewrites an image's configuration as it loads, and an")
say(" id is that configuration's digest. The bundle names what the")
say(" runtime answers for " + loaded.Tag + " afterwards, which a dry run")
say(" cannot ask for without loading.")
return loaded, nil
}
// Through a file rather than through stdin: the runner this repository shares runs a command
// and captures its output, and giving it a second mouth for one caller would change every
// applier's contract for the sake of one step (internal/apply's Runner).
tarball, err := os.CreateTemp("", "mesh-control-*.tar")
if err != nil {
return loaded, fmt.Errorf("nowhere to put the carried image while loading it: %w", err)
}
defer os.Remove(tarball.Name())
if _, err := tarball.Write(saved); err != nil {
tarball.Close()
return loaded, fmt.Errorf("cannot write the carried image to %s: %w", tarball.Name(), err)
}
if err := tarball.Close(); err != nil {
return loaded, fmt.Errorf("cannot finish writing %s: %w", tarball.Name(), err)
}
if _, err := run(ctx, "docker", "load", "--input", tarball.Name()); err != nil {
return loaded, fmt.Errorf(
"the container runtime would not load the carried control-plane image: %w", err)
}
// Read back, and THIS is the answer the bundle is rewritten to.
after, err := idOfImage(ctx, run, loaded.Tag)
if err != nil {
return loaded, err
}
if after == "" {
return loaded, fmt.Errorf(
"the load reported success and this machine holds nothing called %s.\n"+
"The bundle names the control plane by the id this runtime assigned, so there is "+
"nothing to name. Check what `docker load` actually took", loaded.Tag)
}
loaded.ID = after
say(" loaded " + after + " as " + loaded.Tag)
sayIfDifferent(say, archiveID, after)
return loaded, nil
}
// sayIfDifferent reports the archive's own id when the runtime chose another.
//
// Said every time it happens, because it is surprising, it is ordinary, and somebody comparing
// this report against `docker images` on the machine the image was built on would otherwise
// conclude that the wrong image had been carried.
func sayIfDifferent(say func(string), archiveID, held string) {
if archiveID == held {
return
}
say(" the archive says " + archiveID)
say(" this runtime stored the same image under a different configuration, " +
"which is ordinary — the bundle names what the machine holds")
}
// idOfImage asks the runtime what it holds under a name, or empty if it holds nothing.
//
// It asks for the id back rather than reading the exit code, because the id is what is wanted and
// an exit code is not it. Absent is an answer and not a failure: every other reason the runtime
// might refuse looks the same from here, which is why preflight proves the runtime answers before
// this runs rather than this trying to tell the two apart from an exit status.
//
// What it will not do is accept an answer that is not an image id. That answer becomes the name
// the bundle applies on a machine with no mesh to check anything against, so it is checked here
// where the refusal can say whose mistake it is.
func idOfImage(ctx context.Context, run Runner, name string) (string, error) {
out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", name)
if err != nil {
return "", nil
}
got := strings.TrimSpace(firstLineOf(out))
if got == "" {
return "", nil
}
if !isImageID(got) {
return "", fmt.Errorf(
"asked what this machine holds as %q, the runtime answered %q, which is not an image "+
"id. The bundle would name the control plane by that answer, and it is refused "+
"rather than written down", name, got)
}
return got, nil
}
internal/bootstrap/load_test.go
+313
View File
@@ -0,0 +1,313 @@
package bootstrap
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"github.com/novox/mesh-host/internal/image"
)
// Docker is never required here. What is being tested is which commands the installer issues and
// what it concludes from the answers, so the runtime is injected the way `internal/apply` injects
// its Runner (novox/hq ADR 0017). Behaviour against a real runtime is proved in the lab.
// asked records every command, so a test can assert that something was NOT run — which is the
// whole of what idempotence means here.
type asked struct {
commands []string
answer func(name string, args []string) (string, error)
}
func (a *asked) run(_ context.Context, name string, args ...string) (string, error) {
a.commands = append(a.commands, strings.TrimSpace(name+" "+strings.Join(args, " ")))
if a.answer == nil {
return "", errors.New("this test did not expect any command to be run")
}
return a.answer(name, args)
}
func (a *asked) ran(fragment string) bool {
for _, command := range a.commands {
if strings.Contains(command, fragment) {
return true
}
}
return false
}
// savedImageFixture builds what `docker save` produces, tagged `mesh-control:test` unless a test
// asks for something else. Pass no tags for an archive saved without one.
func savedImageFixture(t *testing.T, digest string, tags ...string) []byte {
t.Helper()
if tags == nil {
tags = []string{"mesh-control:test"}
}
entries, err := json.Marshal([]struct {
Config string
RepoTags []string
}{{Config: digest + ".json", RepoTags: tags}})
if err != nil {
t.Fatal(err)
}
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
if err := writer.WriteHeader(&tar.Header{
Name: "manifest.json", Mode: 0o644, Size: int64(len(entries)),
}); err != nil {
t.Fatal(err)
}
if _, err := writer.Write(entries); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
return buffer.Bytes()
}
// fixtureDigest is what the ARCHIVE calls the image, and runtimeDigest is what a runtime calls it
// after loading the same bytes. They differ on purpose, because they differ in reality: an image
// id is the digest of the image's configuration, and a runtime rewrites that configuration as it
// loads. Measured on a live raise, `mesh-control:development` was `sha256:b86bb81c…` on the
// workstation that saved it and `sha256:2dc21904…` on the machine that loaded it.
const (
fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333"
runtimeDigest = "4444444444444444444444444444444444444444444444444444444444444444"
)
// **The id the bundle is named by comes from the RUNTIME, not from the archive.**
//
// This is the test for the fault that took the lab down. The installer used to read the id out of
// the carried tar and use it for the bundle, which is correct on the machine the image was built
// on and wrong on every machine it is carried to — and a bundle naming an id the machine does not
// hold names an image nothing can serve, because an image named by the digest of its own
// configuration is by definition served by nobody. The apply then stops inside a pull that cannot
// succeed, three steps from the cause.
//
// So the image is identified by its TAG, which survives save and load unchanged, and the runtime
// is asked what that tag resolves to.
func TestTheIdComesFromTheRuntimeAndNotFromTheArchive(t *testing.T) {
runtime := &asked{}
inspected := 0
runtime.answer = func(_ string, args []string) (string, error) {
switch {
case len(args) > 1 && args[0] == "image" && args[1] == "inspect":
inspected++
if inspected == 1 {
// Nothing held yet.
return "", errors.New("Error: No such image")
}
// Loaded — and stored under a configuration of the runtime's own making.
return "sha256:" + runtimeDigest + "\n", nil
case len(args) > 0 && args[0] == "load":
return "Loaded image: mesh-control:test\n", nil
}
return "", fmt.Errorf("unexpected command: %v", args)
}
var said []string
loaded, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) })
if err != nil {
t.Fatal(err)
}
if loaded.ID != "sha256:"+runtimeDigest {
t.Errorf("the bundle would name %q; this machine holds sha256:%s", loaded.ID, runtimeDigest)
}
if loaded.Archive != "sha256:"+fixtureDigest {
t.Errorf("the archive's own id is reported as %q", loaded.Archive)
}
if loaded.Predicted {
t.Error("a real run reported its id as a prediction")
}
// The runtime was asked BY THE TAG, which is the only name that survives the transfer.
if !runtime.ran("docker image inspect --format {{.Id}} mesh-control:test") {
t.Errorf("the runtime was never asked what the tag resolves to: %v", runtime.commands)
}
// And the difference is said out loud, or somebody comparing this against `docker images` on
// the build machine concludes the wrong image was carried.
if !strings.Contains(strings.Join(said, "\n"), "the archive says") {
t.Errorf("nothing was said about the two ids differing: %v", said)
}
}
// A machine that already holds the image is not loaded again, and says so.
//
// This is the idempotence the installer's usefulness rests on: it is run over and over while
// somebody gets a machine working, and a step that did its work again every time would be
// indistinguishable from one that had never run. **Decided from what the runtime holds under the
// tag, not from what the archive predicts** — a predicted id cannot answer this question at all on
// a machine whose runtime rewrites configurations.
func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if len(args) > 1 && args[0] == "image" && args[1] == "inspect" {
return "sha256:" + runtimeDigest + "\n", nil
}
return "", fmt.Errorf("unexpected command: %v", args)
}}
var said []string
loaded, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) })
if err != nil {
t.Fatal(err)
}
if !loaded.Held {
t.Error("the machine already held the image and the load did not say so")
}
if loaded.ID != "sha256:"+runtimeDigest {
t.Errorf("the id reported for an already-held image is %q, and the machine holds sha256:%s",
loaded.ID, runtimeDigest)
}
if runtime.ran("docker load") {
t.Errorf("the image was loaded again although the machine held it: %v", runtime.commands)
}
if !strings.Contains(strings.Join(said, "\n"), "already held") {
t.Errorf("nothing was said about finding the image already there: %v", said)
}
}
// A load that reported success and left nothing there is a failure, not a convergence
// (novox/hq ADR 0018). Without the read-back it would surface later as the host refusing a bundle
// naming an image nothing serves — a true message about the wrong thing.
func TestALoadThatLeftNothingBehindIsAFailure(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if len(args) > 1 && args[0] == "image" && args[1] == "inspect" {
return "", errors.New("Error: No such image")
}
return "Loaded image: mesh-control:test\n", nil
}}
_, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), false, func(string) {})
if err == nil {
t.Fatal("a load that left nothing on the machine was reported as success")
}
// Named by the tag, because that is what was asked about and what is missing.
if !strings.Contains(err.Error(), "mesh-control:test") {
t.Errorf("the failure does not say what this machine holds nothing of: %v", err)
}
}
// **A dry run cannot know the id, and says so rather than pretending.** It loads nothing, so no
// runtime has decided anything, and the id in the archive is a fact about a file rather than a
// prediction about this machine. `--dry-run` still produces and checks the bundle's shape; what it
// cannot promise is the one value that only a load can settle.
func TestADryRunLoadsNothingAndSaysTheIdIsUnconfirmed(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if len(args) > 1 && args[0] == "image" && args[1] == "inspect" {
return "", errors.New("Error: No such image")
}
return "", fmt.Errorf("a dry run ran %v", args)
}}
var said []string
loaded, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), true, func(line string) { said = append(said, line) })
if err != nil {
t.Fatal(err)
}
if !loaded.Predicted {
t.Error("a dry run reported an id no runtime had confirmed as though it had been")
}
if loaded.ID != "sha256:"+fixtureDigest {
t.Errorf("a dry run reported %q, and the archive says sha256:%s", loaded.ID, fixtureDigest)
}
if runtime.ran("docker load") {
t.Errorf("a dry run loaded an image: %v", runtime.commands)
}
if !strings.Contains(strings.Join(said, "\n"), "NOT THE FINAL ID") {
t.Errorf("a dry run did not say its id is unconfirmed: %v", said)
}
}
// A dry run on a machine that already holds the image DOES know the id, because the runtime was
// asked and answered. Reading is not changing, so a dry run is entitled to that.
func TestADryRunOnAMachineThatHoldsItKnowsTheRealId(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if len(args) > 1 && args[0] == "image" && args[1] == "inspect" {
return "sha256:" + runtimeDigest + "\n", nil
}
return "", fmt.Errorf("a dry run ran %v", args)
}}
loaded, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), true, func(string) {})
if err != nil {
t.Fatal(err)
}
if loaded.Predicted {
t.Error("an id this machine's runtime supplied was reported as a prediction")
}
if loaded.ID != "sha256:"+runtimeDigest {
t.Errorf("the id is %q, and the runtime said sha256:%s", loaded.ID, runtimeDigest)
}
}
// **An untagged archive is a build-time fault, refused rather than worked around.** Without a tag
// there is no portable name to ask the runtime about, and the only thing left is scraping the
// sentence `docker load` prints for a person — which differs between runtime versions and is
// exactly the kind of guess this whole step exists to stop making.
func TestAnUntaggedArchiveIsRefused(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
return "", fmt.Errorf("nothing should have been run: %v", args)
}}
_, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest, []string{}...), false, func(string) {})
if err == nil {
t.Fatal("an archive with no tag was accepted, and there is no way to ask about it")
}
if !strings.Contains(err.Error(), "make bootstrap") {
t.Errorf("the refusal does not say how to build one that is tagged: %v", err)
}
if len(runtime.commands) != 0 {
t.Errorf("the machine was touched first: %v", runtime.commands)
}
}
// An installer built from a plain checkout carries no image, and says which build step is missing.
// Discovered here, before a machine is touched, rather than after a bundle has been written.
func TestAnInstallerCarryingNoImageSaysSoRatherThanRaisingHalfAMesh(t *testing.T) {
if !image.IsEmpty() {
t.Skip("this checkout has a saved image embedded")
}
_, err := Load(context.Background(), (&asked{}).run, false, func(string) {})
if !errors.Is(err, image.ErrEmpty) {
t.Fatalf("an installer with no control-plane image gave %v, want ErrEmpty", err)
}
if !strings.Contains(err.Error(), "make bootstrap") {
t.Errorf("the refusal does not say how to build one that carries an image: %v", err)
}
}
// An answer that is not an image id is refused rather than written into a bundle.
//
// **This replaces a test that refused an answer differing from the archive's id.** That test
// encoded the mistake: a differing id is now the expected case, not a fault, because a runtime
// rewrites an image's configuration as it loads. What is still worth refusing is an answer that is
// not an id at all — that value becomes the name a bundle applies on a machine with no mesh to
// check anything against, so it is checked where the refusal can say whose mistake it is.
func TestARuntimeAnsweringSomethingThatIsNotAnImageIdIsRefused(t *testing.T) {
for _, nonsense := range []string{
"mesh-control:test",
"sha256:" + strings.Repeat("9", 63),
"<no value>",
} {
runtime := &asked{answer: func(_ string, _ []string) (string, error) {
return nonsense + "\n", nil
}}
if _, err := loadImage(context.Background(), runtime.run,
savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil {
t.Errorf("the runtime answered %q and it was accepted as an image id", nonsense)
}
}
}
internal/bootstrap/module.go
+145
View File
@@ -0,0 +1,145 @@
package bootstrap
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
)
// placeholderDigest is what the catalogue writes where a built image's digest will go.
//
// Sixty-four zeros. A manifest in the catalogue names an image the mesh builds and pushes, and
// until that has happened there is no digest to name — so the convention is a digest that is
// obviously not one, replaced by the pipeline when it publishes. The installer meets it twice: it
// must NOT be there in the registry's manifest, whose image is upstream and never built
// (novox/hq 04-ISSUES/029), and it MUST be there in the control plane's, which is the image
// step 8 has just pushed.
const placeholderDigest = "sha256:" + "0000000000000000000000000000000000000000000000000000000000000000"
// catalogueDir is where a mesh-catalog checkout keeps its manifests.
const catalogueDir = "modules"
// manifestFile is the path a module's manifest is read from, given a catalogue checkout.
func manifestFile(catalogue, module string) string {
return filepath.Join(catalogue, catalogueDir, module, "module.json")
}
// readManifest reads one module's manifest out of a mesh-catalog checkout.
//
// **From a checkout rather than from anything the mesh serves**, and that is the ordering the whole
// pivot exists to respect: at this point the mesh has no build machine, no forge and — until step 7
// finishes — no registry. What a manifest is, is a file; the installer is handed the directory it
// is in and reads it.
func readManifest(catalogue, module string) ([]byte, error) {
path := manifestFile(catalogue, module)
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf(
"the %s module's manifest could not be read: %w\n"+
"--catalog is a checkout of the mesh's catalogue repository, and this is read from "+
"%s inside it", module, err, filepath.Join(catalogueDir, module, "module.json"))
}
return raw, nil
}
// Installed is what installing one module did.
type Installed struct {
// Module is its name.
Module string
// Known is true when the mesh already had this module in its catalogue. The manifest is
// registered either way — a re-run with a changed manifest must land — so this reports what
// was found rather than what was skipped.
Known bool
// Assigned is true when this node was given the module during this run.
Assigned bool
// Pushed is what the mesh said when it sent this node its declaration.
Pushed string
}
// installModule registers a manifest, gives it to this node, and sends it.
//
// The three verbs a person types, in the order they type them, through the same commands. There is
// no installer-only path into the mesh: everything here is `module add`, `assign` and `push`, so
// what the installer does on a bare machine and what an operator does on a running mesh are the
// same act (novox/hq ADR 0035, one refusal per act however it is asked for).
func installModule(ctx context.Context, o Options, control controlPlane, module string,
manifest []byte, say func(string)) (Installed, error) {
out, err := registerAndAssign(ctx, o, control, module, manifest, say)
if err != nil {
return out, err
}
out.Pushed, err = pushNode(ctx, o, control, say)
return out, err
}
// registerAndAssign is the half of installing that happens before anything is sent.
//
// Split out because one module needs something in between: the control plane's own store
// connections have to be accepted before its declaration is composed, or the mesh would seal
// thirty-two random bytes into the file it expects a connection string in and the container would
// come up unable to open anything (mesh-control's `secret accept`, and what it exists for).
//
// **`module add` is run every time and is not skipped when the module is already known.** It is an
// upsert on the manifest, and the manifest is exactly what changes between runs — step 9 registers
// the control plane with a digest that did not exist the first time. Skipping it because the name
// was already in the catalogue would silently pin the mesh to the previous image.
func registerAndAssign(ctx context.Context, o Options, control controlPlane, module string,
manifest []byte, say func(string)) (Installed, error) {
out := Installed{Module: module}
known, err := control.tell(ctx, "module", "list")
if err != nil {
return out, err
}
out.Known = mentions(known, module)
remote := "/" + module + "-module.json"
if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
return out, err
}
if _, err := control.tell(ctx, "module", "add", remote); err != nil {
return out, err
}
if out.Known {
say(" registered " + module + " — the mesh already knew it; the manifest is now this one")
} else {
say(" registered " + module)
}
assigned, err := control.tell(ctx, "assign", o.Node, module)
if err != nil {
return out, err
}
out.Assigned = true
say(" assigned " + module + " to " + o.Node)
if refusal := strings.TrimSpace(assigned); strings.Contains(refusal, "but ") {
// `assign` records what a person meant and says at once when the machine cannot host it.
// Repeated rather than swallowed: the push below will apply everything else and this is
// the only place the reason appears.
say(indent(refusal))
}
return out, nil
}
// pushNode sends this node everything it should be.
//
// Given the long wait rather than the probe timeout: a push composes every declaration this node
// should hold, seals every secret in them and publishes them, and on a first node that is the
// slowest thing the mesh does.
func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
said, err := control.within(o.Wait).tell(ctx, "push", o.Node)
if err != nil {
return "", fmt.Errorf(
"%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+
"not been sent it. Nothing is half-applied — a push the mesh refused sent nothing "+
"at all. Fix what it named and run this installer again: it will find the module "+
"already registered and try the push again", err)
}
say(" pushed " + o.Node)
return strings.TrimSpace(said), nil
}
internal/bootstrap/preflight.go
+240
View File
@@ -0,0 +1,240 @@
package bootstrap
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/image"
"github.com/novox/mesh-host/internal/profile"
)
// DefaultRegistry is where an image reference that names no host comes from.
const DefaultRegistry = "registry-1.docker.io:443"
// Preflight refuses early and plainly, and returns the bundle template it read.
//
// Everything here is a thing that will otherwise be discovered half way through: a machine with
// no runtime found after a bundle has been written, a template that does not parse found after an
// image has been loaded, a registry that cannot be reached found inside a `docker pull` that
// reports a network error and not a missing image. The order is cheapest first, so a mistake in
// what the installer was pointed at costs nothing to find.
func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte, error) {
// 1. Does this installer carry what it claims to?
//
// Asked before the machine is touched, for the same reason `mesh-host bundle` exists: a host
// that carries no substrate must say so when somebody asks, not on a first node
// (internal/bundle). An installer built without an image would otherwise get a machine as far
// as a running store and a running broker and stop.
if image.IsEmpty() {
return nil, image.ErrEmpty
}
saved, err := image.Saved()
if err != nil {
return nil, err
}
//
// The id said here is the ARCHIVE's, and it is reported as such: it is a fact about the file
// and not about this machine. What this runtime will call the image once it holds it is the
// runtime's decision, made at the load, and asked for there (see Load).
carriedID, err := image.ArchiveID(saved)
if err != nil {
return nil, err
}
tag := firstOr(image.Tags(saved), "")
if tag == "" {
// Refused here as well as at the load, because preflight's whole job is to find at the
// start what would otherwise be found half way through — and this would be found after an
// image had been written to disk and handed to a container runtime.
return nil, fmt.Errorf(
"the carried control-plane image has no tag, and the installer identifies it by one: "+
"an image id is the digest of a configuration that a runtime rewrites as it loads, "+
"so the archive's id (%s) is not necessarily the id this machine would hold.\n"+
"Rebuild the installer with a tagged image: `make bootstrap IMAGE=<name>:<tag>`",
carriedID)
}
say(fmt.Sprintf(" control plane %s carried (the archive calls it %s)", tag, carriedID))
// 2. Is the template there, and is it a substrate?
template, err := os.ReadFile(o.Template)
if err != nil {
return nil, fmt.Errorf(
"the bundle template could not be read: %w\n"+
"It is what this machine will be asked to be, so there is nothing to do without "+
"it. Point --bundle at one; mesh-host's examples/substrate-first-node.lock is "+
"the shape", err)
}
// Parsed here as well as at the rewrite, because a template that is not a declaration should
// cost a second rather than an image load and a written file.
parsed, err := declaration.ParseFileTrusted(template)
if err != nil {
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
if _, err := controlPlaneIn(parsed); err != nil {
return nil, err
}
say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources)))
// 3. Does a container runtime ANSWER?
//
// Not "is it installed" — novox/hq 04-ISSUES/007 is exactly that mistake, and the detector
// this uses is the one written for it: it asks the daemon for its server version, which fails
// when the daemon is down however complete the installation is.
//
// **Yes, the bundle installs the runtime itself**, and that is not a contradiction. The
// installer needs one BEFORE the apply, because the control plane's image is loaded into it
// first; the bundle still declares the package and the service because the host must own them
// and reassert them at every reconcile. So this is not a duplicate check — it is the one thing
// the bootstrap cannot bootstrap.
//
// Polled rather than asked once. A socket-activated daemon queued behind
// `network-online.target` is not absent, it is a few seconds away, and `docker load` against
// one blocks silently rather than failing (04-ISSUES/024). Waiting is the honest reading.
if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil {
return nil, err
}
// 4. Can this machine reach what the bundle's images come from?
//
// Asked of the hosts the bundle actually names rather than of the internet in general. The
// mesh's own image is carried and needs nothing served — it is skipped here for exactly that
// reason. Everything else is somebody else's image at somebody else's registry, and a machine
// that cannot reach it fails inside a pull, which reports a network error where a person
// reads a missing image.
//
// "The mesh's own image is skipped" used to mean "skipped if the template happened to name it
// in a way that needs no registry", and that is not the same sentence. A template names the
// control plane by SOMETHING — the reference is a slot, and step 3 replaces whatever is in it
// with the id of the image this installer carries. Whatever the slot held is therefore never
// pulled, never fetched, and never reached; requiring it to be reachable refuses a correct
// install because of a string that is about to be thrown away. Found on the first real run: the
// lab's template still carried `192.0.2.250:5000/mesh-control@…`, the address of a registry that
// no longer exists, and preflight timed out dialling it.
for _, host := range registriesIn(parsed) {
dialing, cancel := context.WithTimeout(ctx, o.Timeout)
err := d.Dial(dialing, host)
cancel()
if err != nil {
return nil, fmt.Errorf(
"this machine cannot reach %s, and the bundle's images are served from there: "+
"%w\nThe apply would fail inside a pull, which says the wrong thing. Fix the "+
"machine's network, or point the bundle at a registry it can reach",
host, err)
}
say(" reachable " + host)
}
return template, nil
}
// waitForRuntime asks the runtime, repeatedly, until it answers or the wait runs out.
func waitForRuntime(ctx context.Context, run Runner, probe, wait time.Duration, say func(string)) error {
detector := containerRuntimeDetector(run)
deadline := time.Now().Add(wait)
var last string
for {
probing, cancel := context.WithTimeout(ctx, probe)
verdict := detector.Detect(probing)
cancel()
if verdict.Present {
say(" container runtime " + verdict.Detail)
return nil
}
last = verdict.Detail
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(runtimeAskEvery):
}
}
return fmt.Errorf(
"this machine has no container runtime that answers, after waiting %s: %s\n"+
"An installed package is not a capability (novox/hq 04-ISSUES/007) — the daemon was "+
"asked and did not reply. Start it, then run this again; every step is idempotent",
wait, last)
}
// runtimeAskEvery is how often the runtime is asked again while waiting for it.
var runtimeAskEvery = 2 * time.Second
// containerRuntimeDetector is the host's OWN detector for a working runtime, not a second
// implementation of the same question. Two answers to "is there a container runtime here" is how
// the installer and the host come to disagree about a machine.
func containerRuntimeDetector(run Runner) profile.Detector {
for _, detector := range profile.Default(profile.Runner(run)) {
if detector.Name() == profile.CapContainerRuntime {
return detector
}
}
// Unreachable unless the host's own detector set loses its container runtime, which would be
// a change nobody would make on purpose — said rather than nil-dereferenced.
panic("the host detects no container runtime capability, and the installer needs that answer")
}
// registriesIn is every host the bundle's images would be fetched from, without duplicates and in
// the order they appear.
//
// The control plane's own resource is excluded by identity rather than by the shape of what it
// names. Its image reference is a slot the installer overwrites with the id of the image it
// carries, so no registry ever serves it — and a template that filled that slot with a registry
// this machine cannot reach is not a machine with a network problem.
func registriesIn(d *declaration.Declaration) []string {
var hosts []string
seen := map[string]bool{}
for _, r := range d.Resources {
container, ok := r.(*declaration.Container)
if !ok || container.Identity() == ControlPlaneID {
continue
}
host, served := registryOf(container.Image)
if !served || seen[host] {
continue
}
seen[host] = true
hosts = append(hosts, host)
}
return hosts
}
// registryOf says where an image would be fetched from, and whether anything has to serve it.
//
// The second return is false for an image named by the digest of its own configuration: nothing
// serves those and nothing can (see `internal/declaration`'s checkImage). That is the whole reason
// the mesh's own control plane can be raised on a machine with no registry anywhere.
//
// The rule for the rest is the container runtime's own: the part before the first slash is a
// registry host if it looks like one — it has a dot, or a port, or it is `localhost` — and
// otherwise it is part of a repository name on the default registry.
func registryOf(reference string) (string, bool) {
if reference == "" || strings.HasPrefix(reference, "sha256:") {
return "", false
}
name := reference
if at := strings.Index(name, "@"); at >= 0 {
name = name[:at]
}
first, _, hasPath := strings.Cut(name, "/")
if !hasPath || !(strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost") {
return DefaultRegistry, true
}
if !strings.Contains(first, ":") {
// A registry with no port is reached over HTTPS, which is where a pull would go.
return first + ":443", true
}
return first, true
}
func firstOr(values []string, fallback string) string {
if len(values) == 0 || strings.TrimSpace(values[0]) == "" {
return fallback
}
return values[0]
}
internal/bootstrap/preflight_test.go
+151
View File
@@ -0,0 +1,151 @@
package bootstrap
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// An installed package is not a capability (novox/hq 04-ISSUES/007). The daemon is asked, and a
// machine where it does not answer is refused before anything is loaded, written or applied.
//
// The refusal has to be plain, because the person reading it is standing in front of a machine
// that will not work: it says what was asked, what came back, that re-running is safe, and names
// the record that explains why an installed docker is not enough.
func TestPreflightRefusesPlainlyWhenTheRuntimeDoesNotAnswer(t *testing.T) {
silent := func(context.Context, string, ...string) (string, error) {
return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
}
// No wait, so this is one attempt: what is being tested is the refusal, not the patience.
err := waitForRuntime(context.Background(), silent, time.Second, 0, func(string) {})
if err == nil {
t.Fatal("a machine whose container runtime does not answer was accepted")
}
for _, wanted := range []string{
"no container runtime that answers",
"Cannot connect to the Docker daemon",
"04-ISSUES/007",
"idempotent",
} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
}
}
}
// And a runtime that is merely slow to start is waited for rather than refused.
//
// A socket-activated daemon queued behind the network is not absent, it is a few seconds away.
// Refusing on the first attempt would make a correct bootstrap fail for being observed too early —
// and `docker load` against such a daemon blocks silently rather than failing, which is how one
// became a 35-minute silence (04-ISSUES/024).
func TestARuntimeThatIsStillStartingIsWaitedFor(t *testing.T) {
previous := runtimeAskEvery
runtimeAskEvery = time.Millisecond
defer func() { runtimeAskEvery = previous }()
attempts := 0
slow := func(context.Context, string, ...string) (string, error) {
attempts++
if attempts < 3 {
return "", errors.New("Cannot connect to the Docker daemon")
}
return "27.0.3\n", nil
}
var said []string
if err := waitForRuntime(context.Background(), slow, time.Second, time.Second,
func(line string) { said = append(said, line) }); err != nil {
t.Fatalf("a runtime that answered on the third ask was refused: %v", err)
}
if attempts != 3 {
t.Errorf("the runtime was asked %d time(s)", attempts)
}
if !strings.Contains(strings.Join(said, "\n"), "27.0.3") {
t.Errorf("the version the daemon reported was not said back: %v", said)
}
}
// What has to be reachable is what the bundle actually names, not "the internet".
//
// The mesh's own image is carried and nothing serves it, so asking a registry about it would be
// asking a question with no answer — which is the whole point of naming an image by the digest of
// its own configuration.
func TestOnlyTheRegistriesTheBundleNamesAreAskedAbout(t *testing.T) {
parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[
{"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` +
strings.Repeat("7", 64) + `"},
{"id":"broker","type":"container","name":"mesh-broker","image":"192.0.2.250:5000/lavinmq@sha256:` +
strings.Repeat("8", 64) + `"},
{"id":"control-plane","type":"container","name":"mesh-control","image":"` + held + `"}
]}`))
if err != nil {
t.Fatal(err)
}
got := registriesIn(parsed)
want := []string{DefaultRegistry, "192.0.2.250:5000"}
if len(got) != len(want) {
t.Fatalf("asked about %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Errorf("asked about %v, want %v", got, want)
}
}
}
// And the control plane's slot is excluded whatever is in it.
//
// Found on the first real run of this installer. A template names the control plane by SOMETHING
// and step 3 replaces it with the id of the carried image, so whatever was there is never pulled —
// but preflight was reading that slot like any other and dialling it. The lab's template still
// carried the address of a registry the lab no longer raises, so a correct install timed out in
// preflight against a machine with a perfectly good network.
func TestTheControlPlanesOwnRegistryIsNeverAskedAbout(t *testing.T) {
parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[
{"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` +
strings.Repeat("7", 64) + `"},
{"id":"control-plane","type":"container","name":"mesh-control","image":"192.0.2.250:5000/mesh-control@sha256:` +
strings.Repeat("8", 64) + `"}
]}`))
if err != nil {
t.Fatal(err)
}
got := registriesIn(parsed)
if len(got) != 1 || got[0] != DefaultRegistry {
t.Fatalf("asked about %v; the control plane's own reference is about to be replaced and "+
"must not be reached for", got)
}
}
func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) {
// The container runtime's own rule: the part before the first slash is a registry host if it
// has a dot, a port, or is localhost. Getting this wrong means dialling a hostname that is
// really the first half of a repository name, and refusing a machine that is fine.
for _, c := range []struct {
reference string
host string
served bool
}{
{"postgres@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true},
{"cloudamqp/lavinmq@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true},
{"192.0.2.250:5000/postgres@sha256:" + strings.Repeat("a", 64), "192.0.2.250:5000", true},
{"localhost/mesh-control@sha256:" + strings.Repeat("a", 64), "localhost:443", true},
{"registry.example.com/a/b@sha256:" + strings.Repeat("a", 64), "registry.example.com:443", true},
// Held by this machine. Nothing serves it, and nothing can.
{"sha256:" + strings.Repeat("a", 64), "", false},
{"", "", false},
} {
host, served := registryOf(c.reference)
if host != c.host || served != c.served {
t.Errorf("%q → (%q, %v), want (%q, %v)", c.reference, host, served, c.host, c.served)
}
}
}
internal/bootstrap/publish.go
+173
View File
@@ -0,0 +1,173 @@
package bootstrap
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// ControlPlaneRepository is what the control plane's image is called in the mesh's own registry.
const ControlPlaneRepository = "mesh-control"
// genesisTag is the tag the first push uses.
//
// A tag is not a pin and is never what anything is deployed from — the digest the registry assigns
// is (novox/hq ADR 0006). This exists so a person reading `/v2/mesh-control/tags/list` can see
// which image this mesh started from, and so the push has something to name. Everything downstream
// uses the digest that comes back.
const genesisTag = "genesis"
// Published is what step 8 did.
type Published struct {
// Reference is `<registry>/mesh-control@sha256:…` — the first manifest digest this image has
// ever had, and the thing that makes the control plane an ordinary module.
Reference string
// Tagged is where it was pushed, tag and all.
Tagged string
// Already is true when the registry was already serving it and nothing was pushed.
Already bool
}
// PublishControlPlane puts the carried image into the mesh's own registry and reads back its digest.
//
// **This is the pivot's hinge.** Every image must be pinned by digest, and a digest a pin can mean
// is one a REGISTRY assigned when something was pushed to it. The control plane's image is built
// from source and pushed nowhere, so it has none — which is why the substrate names it by the
// digest of its own configuration, and why that is legal exactly where nothing could have served
// one. The moment this push completes, that stops being true: the image has a manifest digest, so
// the control plane can be named the way every other module is named, so the mesh can build and
// roll out its own upgrades. If this step is skipped the machine still works and the mesh cannot
// upgrade itself, which is the check novox/hq ADR 0067 states: after installing, the running
// control plane must be pinned by a digest the mesh's own registry assigned, not by an image id.
//
// **It mirrors mesh-control's `internal/builder`.PublishImage rather than importing it.** Tag,
// push, read back `RepoDigests`, refuse anything without `@sha256:` — the same four steps, because
// there is exactly one right way to learn what a registry will serve something as, and it is to
// ask the registry. It is not imported because that code is tier 2: the host and its installer
// depend on nothing that must be installed first (novox/hq ADR 0041), and taking a dependency on
// the control plane's repository to raise the control plane would be the cycle this whole ADR is
// about, one layer up. The duplication is four commands, and it is deliberate.
//
// One difference, and it is a correction rather than a divergence: the digest is chosen from
// `RepoDigests` by repository instead of taken as element zero. An image that has been pushed to
// more than one registry has more than one entry, and element zero is then whichever the runtime
// happened to list first — which would pin this mesh to somebody else's registry, silently.
func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string,
say func(string)) (Published, error) {
remote := o.Registry + "/" + ControlPlaneRepository
out := Published{Tagged: remote + ":" + genesisTag}
// Asked first. A digest already served is a fact about the registry, and re-pushing an image
// the registry already holds is asking it to store what it already has under the name it
// already has.
if held, err := digestOf(ctx, o, d, remote); err != nil {
return out, err
} else if held != "" {
out.Reference, out.Already = held, true
say(" already published " + held)
return out, nil
}
if _, err := d.Run(ctx, "docker", "tag", imageID, out.Tagged); err != nil {
return out, fmt.Errorf("cannot tag the carried image as %s: %w", out.Tagged, err)
}
if _, err := d.Run(ctx, "docker", "push", out.Tagged); err != nil {
return out, fmt.Errorf(
"the container runtime would not push %s: %w\n"+
"The registry is plain HTTP and wants no credentials, deliberately — it is reached "+
"over the mesh's own network, which is already the encrypted and authenticated "+
"thing. A runtime refusing it for being insecure is refusing a registry on %s, "+
"which it does not do for a loopback address",
out.Tagged, err, o.Registry)
}
// Read back, from the registry's own answer rather than computed here. What matters is what
// the registry will serve for that reference, and only it can say (novox/hq ADR 0018).
pinned, err := digestOf(ctx, o, d, remote)
if err != nil {
return out, err
}
if pinned == "" {
return out, fmt.Errorf(
"%s was pushed and the registry does not serve it.\n"+
"The next step names the control plane's module by the digest this was supposed to "+
"produce, so there is nothing to name. Check `docker push` and "+
"http://%s/v2/%s/tags/list", out.Tagged, o.Registry, ControlPlaneRepository)
}
out.Reference = pinned
say(" published " + pinned)
return out, nil
}
// digestOf is what the registry serves this repository as, or empty if it serves it at all.
//
// Both halves are asked, because either alone lies. The registry's tag list says something was
// pushed and not what its digest is; the runtime's `RepoDigests` says what a digest was and not
// whether the registry still has it — a registry whose volume was recreated would leave the
// runtime remembering a digest nothing serves, and the module registered against it would pin the
// mesh to an image that cannot be pulled.
func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) {
asking, cancel := context.WithTimeout(ctx, o.Timeout)
status, body, err := d.Fetch(asking,
"http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list")
cancel()
if err != nil {
return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err)
}
if status == http.StatusNotFound {
// Nothing has ever been pushed under this name. An answer, not a failure.
return "", nil
}
if status != http.StatusOK {
return "", fmt.Errorf("the registry answered %d when asked what it holds for %s",
status, ControlPlaneRepository)
}
var listed struct {
Tags []string `json:"tags"`
}
if err := json.Unmarshal([]byte(body), &listed); err != nil {
return "", fmt.Errorf("the registry's answer about %s is not readable: %w",
ControlPlaneRepository, err)
}
if !contains(listed.Tags, genesisTag) {
return "", nil
}
// The registry has it. What digest, according to the runtime that pushed it.
reading, cancel := context.WithTimeout(ctx, o.Timeout)
out, err := d.Run(reading, "docker", "inspect", "--format", "{{json .RepoDigests}}",
remote+":"+genesisTag)
cancel()
if err != nil {
// The registry holds the tag and this machine's runtime does not hold the image. That
// happens on a re-run after the image was pruned, and it is not something to work around
// by trusting the tag: a tag can be made to point elsewhere.
return "", nil
}
var digests []string
if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &digests); err != nil {
return "", fmt.Errorf("the runtime's answer about %s is not readable: %w", remote, err)
}
for _, digest := range digests {
if !strings.HasPrefix(digest, remote+"@sha256:") {
// Somebody else's registry serving the same image. Skipped rather than used: pinning
// this mesh's control plane to a registry it does not run is exactly the dependency
// the pivot exists to remove.
continue
}
return digest, nil
}
return "", nil
}
func contains(values []string, want string) bool {
for _, v := range values {
if v == want {
return true
}
}
return false
}
internal/bootstrap/publish_test.go
+197
View File
@@ -0,0 +1,197 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"testing"
"time"
)
// Step 8 is the pivot's hinge (novox/hq ADR 0067): the carried image gets a manifest digest, which
// is the first one it has ever had, and that is what lets the control plane be named the way every
// other module is named. These tests defend how that digest is learned, because a wrong one pins
// the mesh to an image nothing on this machine serves.
func publishing(t *testing.T, fetch func(string) (int, string, error),
run func(name string, args []string) (string, error)) (Options, Deps, *asked) {
t.Helper()
runtime := &asked{answer: run}
return Options{
Registry: "127.0.0.1:5000",
Timeout: time.Second,
Wait: 0,
}, Deps{
Run: runtime.run,
Fetch: func(_ context.Context, url string) (int, string, error) {
return fetch(url)
},
}, runtime
}
// Nothing has ever been pushed under this name, so the registry says 404 — and that is an answer,
// not a failure. An installer that treated it as one would refuse on the first run of the step it
// exists to perform.
func TestAnImageNoRegistryHasEverHeldIsPushed(t *testing.T) {
pushed := false
o, d, runtime := publishing(t,
func(string) (int, string, error) {
if !pushed {
return http.StatusNotFound, "", nil
}
return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil
},
func(_ string, args []string) (string, error) {
switch args[0] {
case "tag":
return "", nil
case "push":
pushed = true
return "", nil
case "inspect":
return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("a", 64) + `"]`, nil
}
return "", fmt.Errorf("unexpected: %v", args)
})
out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Already {
t.Error("an image no registry held was reported as already published")
}
if !strings.HasPrefix(out.Reference, "127.0.0.1:5000/mesh-control@sha256:") {
t.Errorf("the control plane is pinned as %q", out.Reference)
}
if !runtime.ran("docker push 127.0.0.1:5000/mesh-control:genesis") {
t.Errorf("nothing was pushed: %v", runtime.commands)
}
}
// An image the registry already serves is not pushed again, and says so. Blobs are named by their
// content, so re-pushing is asking a registry to store what it already has under the name it
// already has — and the installer is run over and over.
func TestAnImageTheRegistryAlreadyServesIsNotPushedAgain(t *testing.T) {
o, d, runtime := publishing(t,
func(string) (int, string, error) {
return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil
},
func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("b", 64) + `"]`, nil
}
return "", fmt.Errorf("unexpected: %v", args)
})
out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err != nil {
t.Fatal(err)
}
if !out.Already {
t.Error("an image the registry already serves was not reported as already published")
}
if runtime.ran("docker push") {
t.Errorf("it was pushed again: %v", runtime.commands)
}
}
// **The digest is chosen by repository, not taken as element zero.** An image that has been pushed
// to more than one registry has more than one entry, and element zero is whichever the runtime
// listed first — which would pin this mesh's control plane to somebody else's registry, silently,
// which is the dependency the whole pivot exists to remove.
func TestTheDigestComesFromThisMeshsOwnRegistry(t *testing.T) {
elsewhere := "some.other.registry/mesh-control@sha256:" + strings.Repeat("c", 64)
ours := "127.0.0.1:5000/mesh-control@sha256:" + strings.Repeat("d", 64)
o, d, _ := publishing(t,
func(string) (int, string, error) {
return http.StatusOK, `{"tags":["genesis"]}`, nil
},
func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return `["` + elsewhere + `","` + ours + `"]`, nil
}
return "", fmt.Errorf("unexpected: %v", args)
})
out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Reference != ours {
t.Errorf("the control plane is pinned as %q, and this mesh's registry serves %q",
out.Reference, ours)
}
}
// A push that produced no digest this mesh's registry serves is refused, and the refusal says what
// depends on it. The next step names the control plane's module by that digest, so there would be
// nothing to name — and finding that out one step later would mean registering a module pinned to
// an empty string.
func TestAPushThatProducedNoDigestIsRefused(t *testing.T) {
pushed := false
o, d, _ := publishing(t,
func(string) (int, string, error) {
if !pushed {
return http.StatusNotFound, "", nil
}
// Pushed, and the registry still does not list it.
return http.StatusOK, `{"tags":[]}`, nil
},
func(_ string, args []string) (string, error) {
if args[0] == "push" {
pushed = true
}
return "", nil
})
_, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err == nil {
t.Fatal("a push that produced no digest was accepted")
}
if !strings.Contains(err.Error(), "does not serve it") {
t.Errorf("the refusal does not say what is missing: %v", err)
}
}
// A tag is not a pin. If the runtime answers with something that is not pinned by digest, it is
// not used — a tag can be made to point at a different image, and this reference is applied on
// machines with no mesh to ask about anything (novox/hq ADR 0006).
func TestATagIsNotAPin(t *testing.T) {
o, d, _ := publishing(t,
func(string) (int, string, error) {
return http.StatusOK, `{"tags":["genesis"]}`, nil
},
func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return `["127.0.0.1:5000/mesh-control:genesis"]`, nil
}
return "", nil
})
out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err == nil {
t.Fatalf("a tag was accepted as a pin: %q", out.Reference)
}
}
// A registry that cannot be reached at all is said so plainly rather than becoming a push that
// fails for a reason nobody can read.
func TestARegistryThatCannotBeAskedIsSaidSo(t *testing.T) {
o, d, _ := publishing(t,
func(string) (int, string, error) {
return 0, "", errors.New("connection refused")
},
func(string, []string) (string, error) { return "", nil })
_, err := PublishControlPlane(context.Background(), o, d, held, func(string) {})
if err == nil {
t.Fatal("a registry that refused the connection was treated as empty")
}
if !strings.Contains(err.Error(), "cannot ask the registry") {
t.Errorf("the refusal does not say the registry could not be asked: %v", err)
}
}
internal/bootstrap/registry.go
+204
View File
@@ -0,0 +1,204 @@
package bootstrap
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
)
// RegistryModule is the module that provides the mesh's artifact store.
const RegistryModule = "registry"
// registryResource is the id of the resource in that module's manifest that runs the registry.
// What the container is CALLED is read from the manifest rather than assumed, because the name is
// the catalogue's to choose and the installer only has to know which resource to wait for.
const registryResource = "store"
// Registry is what step 7 did.
type Registry struct {
Installed
// Container is the container the manifest declares, confirmed running.
Container string
// Address is host:port the registry answers on, as this machine reaches it.
Address string
// Answered is the status the registry's own `/v2/` gave back.
Answered int
}
// InstallRegistry gives this mesh somewhere to put images.
//
// **Its image is upstream and it is never built.** novox/hq 04-ISSUES/029 is the whole reason this
// step exists in this position: a module that provides the artifact store cannot be delivered
// through the artifact store, so the registry is the one module whose image is pulled from the
// internet like the store and the broker before it. A manifest carrying the catalogue's
// placeholder digest here would mean somebody had made it buildable, which is the cycle again —
// so it is refused rather than pulled.
//
// **No credentials, and that is deliberate.** The registry is reached over the mesh's own private
// network, which is already the encrypted and authenticated thing; a second layer inside it would
// be certificates to issue and rotate for no property the first does not have (mesh-control's
// `internal/builder`, which pushes to it the same way). So there is nothing here to configure and
// nothing to seal — which is also why step 8 can push without the mesh having issued anything.
//
// **It is verified by asking it, not by looking at it.** A container that is up is not a registry
// that serves: `/v2/` is the registry API's own "yes, I am one and I am ready", and it is the
// question step 8 depends on the answer to.
func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlane,
say func(string)) (Registry, error) {
out := Registry{Address: o.Registry}
manifest, err := readManifest(o.Catalogue, RegistryModule)
if err != nil {
return out, err
}
container, image, err := containerIn(manifest, registryResource)
if err != nil {
return out, err
}
if strings.Contains(image, placeholderDigest) {
return out, fmt.Errorf(
"the %s module's image is %q, which is the catalogue's placeholder for something the "+
"mesh builds and pushes.\n"+
"This module is the one that cannot work that way: it PROVIDES the place built "+
"images go, so it can never be delivered through it (novox/hq 04-ISSUES/029). Its "+
"image is upstream and pinned in the manifest", RegistryModule, image)
}
out.Container = container
say(" registry image " + image + " — upstream, never built")
installed, err := installModule(ctx, o, control, RegistryModule, manifest, say)
out.Installed = installed
if err != nil {
return out, err
}
// Read back, in two stages, because they fail differently. A container that never appears is
// a declaration that did not reach this node or an image that would not pull; a container
// that is up and does not answer is a registry that started and failed.
if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil {
return out, err
}
status, err := waitForTheRegistry(ctx, d, o, say)
if err != nil {
return out, err
}
out.Answered = status
return out, nil
}
// waitForTheRegistry asks `/v2/` until it answers.
func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) {
where := "http://" + o.Registry + "/v2/"
deadline := time.Now().Add(o.Wait)
var last string
for {
asking, cancel := context.WithTimeout(ctx, o.Timeout)
status, _, err := d.Fetch(asking, where)
cancel()
switch {
case err != nil:
last = err.Error()
case status == http.StatusOK:
say(fmt.Sprintf(" replies %s answered %d", where, status))
return status, nil
default:
// A registry that answers 401 is one that wants credentials, which this one is
// configured not to. Reported as what it said rather than retried into a timeout.
last = fmt.Sprintf("it answered %d", status)
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return 0, ctx.Err()
case <-time.After(answerEvery):
}
}
return 0, fmt.Errorf(
"the registry's container is running and %s does not answer, after waiting %s: %s\n"+
"Running is not serving. `/v2/` is the registry API saying it is ready, and the next "+
"step pushes the control plane's image to it — so this is refused here rather than "+
"discovered inside a `docker push`. `docker logs mesh-registry` says what it did",
where, o.Wait, last)
}
// waitForContainer waits for a container the mesh was asked to create to be running.
//
// Unlike the substrate's own verify, this one waits: the mesh applies through a node's host, over
// the broker, asynchronously. A push that the control plane accepted has not yet happened on the
// machine, and refusing on the first look would refuse every correct install.
func waitForContainer(ctx context.Context, run Runner, probe, wait time.Duration, name string,
say func(string)) error {
deadline := time.Now().Add(wait)
var last string
for {
state, err := containerRunning(ctx, run, probe, name)
switch {
case err != nil:
last = "it is not there at all"
case state.running:
say(" running " + name)
return nil
default:
last = "it is " + state.status
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(answerEvery):
}
}
return fmt.Errorf(
"the mesh accepted the push and %q is not running after %s: %s\n"+
"The control plane sends a declaration over the broker and this node's host applies "+
"it, so the two ends fail differently: `mesh-host` on this machine says what it made "+
"of the declaration, and `status` on the control plane says whether it was collected "+
"at all", name, wait, last)
}
// containerIn finds one container resource in a module manifest and gives back its name and image.
//
// It reads the manifest as data rather than through the catalogue's own parser, because that
// parser lives in the control plane and the host depends on nothing installed first
// (novox/hq ADR 0041) — importing it would put tier 2 inside tier 0. What is read here is two
// fields of a shape the catalogue owns; the manifest is handed to the control plane unchanged, and
// it is the control plane's `module add` that judges whether it is a manifest at all.
func containerIn(manifest []byte, id string) (name, image string, err error) {
var m struct {
Module string `json:"module"`
Resources []struct {
ID string `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Image string `json:"image"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return "", "", fmt.Errorf("this manifest is not readable as JSON: %w", err)
}
var containers []string
for _, r := range m.Resources {
if r.Type != "container" {
continue
}
containers = append(containers, r.ID)
if r.ID == id {
return r.Name, r.Image, nil
}
}
return "", "", fmt.Errorf(
"the %s module declares no container %q, so the installer does not know what to wait for. "+
"It declares: %s", m.Module, id, strings.Join(containers, ", "))
}
internal/bootstrap/registry_test.go
+216
View File
@@ -0,0 +1,216 @@
package bootstrap
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// Step 7 installs the one module whose image can never come from the mesh's own registry, because
// it IS the mesh's own registry (novox/hq 04-ISSUES/029). These tests defend that, and defend the
// distinction the whole verify layer of this program is built on: a container that is up is not a
// service that answers.
// catalogueWith writes a fake catalogue checkout holding one module's manifest.
//
// A fixture here rather than the real catalogue, unlike the substrate example the rewrite tests
// use: the catalogue is a different repository on a different branch, and a test that read it
// would pass or fail according to what somebody else had checked out.
func catalogueWith(t *testing.T, module, manifest string) string {
t.Helper()
root := t.TempDir()
dir := filepath.Join(root, catalogueDir, module)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
return root
}
const upstreamRegistryManifest = `{
"module": "registry",
"version": "1",
"provides": [{"name": "artifact-store", "scope": "mesh"}],
"capabilities": ["container-runtime"],
"resources": [
{"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"},
{"id": "store", "type": "container", "name": "mesh-registry",
"image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373",
"ports": ["5000:5000"]}
]
}`
// aMeshThatAgrees answers every command the installer issues at steps 7 and 9 the way a working
// mesh would, except for whatever a test overrides.
func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, error) {
return func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
for fragment, said := range answers {
if strings.Contains(joined, fragment) {
return said, nil
}
}
switch {
case name != "docker":
return "", fmt.Errorf("unexpected program %q", name)
case args[0] == "cp":
return "", nil
case args[0] == "inspect":
return "true running\n", nil
case args[0] == "exec":
return "", nil
}
return "", fmt.Errorf("unexpected: %v", args)
}
}
func installing(t *testing.T, catalogue string) Options {
t.Helper()
return Options{
Node: "anchor",
Catalogue: catalogue,
Registry: "127.0.0.1:5000",
Timeout: time.Second,
Wait: 0,
}
}
// A container that is up is not a registry that serves. `/v2/` is the registry API's own "yes, I
// am one and I am ready", and the step after this pushes to it — so it is refused here rather than
// discovered inside a `docker push`.
func TestARegistryContainerThatIsUpIsNotARegistryThatServes(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
runtime := &asked{answer: aMeshThatAgrees(nil)}
deps := Deps{
Run: runtime.run,
Fetch: func(context.Context, string) (int, string, error) {
return http.StatusInternalServerError, "", nil
},
}
_, err := InstallRegistry(context.Background(),
installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)),
deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err == nil {
t.Fatal("a registry whose container is up and which answers 500 was accepted")
}
for _, wanted := range []string{"/v2/", "Running is not serving"} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
}
}
}
// The whole of step 7, against a mesh that agrees: registered, assigned, pushed, up, and answering.
func TestARegistryThatAnswersIsAccepted(t *testing.T) {
runtime := &asked{answer: aMeshThatAgrees(nil)}
deps := Deps{
Run: runtime.run,
Fetch: func(context.Context, string) (int, string, error) {
return http.StatusOK, "{}", nil
},
}
out, err := InstallRegistry(context.Background(),
installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)),
deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second},
func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Container != "mesh-registry" {
t.Errorf("the registry's container is %q", out.Container)
}
if out.Answered != http.StatusOK {
t.Errorf("the registry answered %d", out.Answered)
}
// Registered, assigned and pushed, through the same three commands a person types.
for _, wanted := range []string{
"module add /registry-module.json",
"assign anchor registry",
"push anchor",
} {
if !runtime.ran(wanted) {
t.Errorf("the installer never ran %q: %v", wanted, runtime.commands)
}
}
}
// **The registry's image is upstream and it is never built.** A manifest carrying the catalogue's
// placeholder digest would mean somebody had made this module buildable — which is the cycle
// novox/hq 04-ISSUES/029 settled: a module that provides the artifact store cannot be delivered
// through the artifact store.
func TestARegistryManifestThatWantsBuildingIsRefused(t *testing.T) {
wants := strings.Replace(upstreamRegistryManifest,
"registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373",
"mesh-runtime-registry@"+placeholderDigest, 1)
runtime := &asked{answer: aMeshThatAgrees(nil)}
_, err := InstallRegistry(context.Background(),
installing(t, catalogueWith(t, RegistryModule, wants)),
Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run},
func(string) {})
if err == nil {
t.Fatal("a registry manifest naming an image the mesh would have to build was accepted")
}
if !strings.Contains(err.Error(), "04-ISSUES/029") {
t.Errorf("the refusal does not name the decision it rests on: %v", err)
}
if runtime.ran("module add") {
t.Error("it was registered anyway")
}
}
// A catalogue that is not there is said plainly, with what --catalog is. This is the most likely
// mistake anybody makes at this step and the least interesting to debug.
func TestACatalogueThatIsNotThereIsSaidPlainly(t *testing.T) {
runtime := &asked{answer: aMeshThatAgrees(nil)}
_, err := InstallRegistry(context.Background(),
installing(t, filepath.Join(t.TempDir(), "nowhere")),
Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run},
func(string) {})
if err == nil {
t.Fatal("a catalogue that does not exist was accepted")
}
if !strings.Contains(err.Error(), "--catalog") {
t.Errorf("the refusal does not say what to fix: %v", err)
}
}
// A refusal from the control plane is repeated verbatim. mesh-control refuses in paragraphs —
// "nothing provides route, wanted by registry" — and an installer that reported "exit status 1"
// would throw away the only thing a person can act on.
func TestWhatTheMeshRefusedIsRepeated(t *testing.T) {
refusal := "nothing provides \"route\", wanted by registry"
runtime := &asked{answer: func(name string, args []string) (string, error) {
if strings.Contains(strings.Join(args, " "), "push") {
return refusal, fmt.Errorf("exit status 1")
}
return aMeshThatAgrees(nil)(name, args)
}}
_, err := InstallRegistry(context.Background(),
installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)),
Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run,
timeout: time.Second}, func(string) {})
if err == nil {
t.Fatal("a push the mesh refused was reported as successful")
}
if !strings.Contains(err.Error(), refusal) {
t.Errorf("what the mesh said is not in the failure:\n%v", err)
}
if !strings.Contains(err.Error(), "run this installer again") {
t.Errorf("the failure does not say a re-run continues from here:\n%v", err)
}
}
internal/bootstrap/retire.go
+291
View File
@@ -0,0 +1,291 @@
package bootstrap
import (
"bytes"
"context"
"fmt"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/system"
)
// Retired is what step 10 did.
type Retired struct {
// Container is the temporary control plane that was dropped.
Container string
// Gone is true when the machine no longer has it.
Gone bool
// Already is true when it was gone before this step ran.
Already bool
// Bundle is where the bundle without it was written.
Bundle string
// Removed is how many resources the re-apply reported removing.
Removed int
}
// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away.
//
// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The
// host owns what it has applied and removes what it owns and is no longer declared. So retiring the
// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the
// bundle is applied again, and the removal pass does what it does for every other resource that
// leaves a declaration.
//
// That is the whole of why the rename at step 3 mattered. Had the substrate and the module both
// called their container `mesh-control`, this apply would have removed the module's container —
// the host would have been asked to take away something it believed it owned, and it would have
// been right. Two names, two owners, and the removal is unambiguous.
//
// **It is the last step for a reason.** Until step 9 has a control plane that answers, the
// temporary one is the only thing that can tell this machine anything, and a machine left with no
// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one
// has been proved, and a run interrupted before it leaves two control planes, which is untidy and
// harmless — a re-run reaches this step and finishes.
func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System,
produced []byte, run Runner, say func(string)) (Retired, error) {
out := Retired{Bundle: o.Out}
current, err := declaration.ParseFileTrusted(produced)
if err != nil {
return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err)
}
temporary, err := controlPlaneIn(current)
if err != nil {
// The bundle already declares no control plane, which is what a re-run after this step
// finds. Nothing to drop, and nothing to be alarmed about.
say(" already dropped the bundle declares no temporary control plane")
out.Already = true
return out, nil
}
out.Container = temporary.Name
// Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be
// READ, and a person coming to a machine after a pivot should be able to open the file the
// installer applied and see the substrate they recognise with the control plane gone from it.
// Re-serialising a parsed declaration would drop every comment in it.
bundle, err := removeResource(produced, ControlPlaneID)
if err != nil {
return out, err
}
without, err := declaration.ParseFileTrusted(bundle)
if err != nil {
return out, fmt.Errorf(
"taking the temporary control plane out of the bundle broke it: %w", err)
}
if _, err := controlPlaneIn(without); err == nil {
return out, fmt.Errorf(
"the bundle still declares %q after it was taken out, so nothing was removed and the "+
"apply below would change nothing", ControlPlaneID)
}
if err := writeBundleFile(o.Out, bundle); err != nil {
return out, err
}
say(fmt.Sprintf(" wrote %s (%d resources) — without %s",
o.Out, len(without.Resources), temporary.Name))
// Applied the same way everything else here is applied, under the same origin, against the
// same state file. What makes this a removal rather than a no-op is that the state file
// records the container as something this installer applied, and the declaration no longer
// asks for it.
report, err := ApplyBundle(ctx, o, sys, without, run, say)
if err != nil {
return out, fmt.Errorf(
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
"That is untidy and it is not broken: two control planes on one mesh are both "+
"stateless and both correct. Run this installer again to finish", err)
}
for _, outcome := range report.Outcomes {
if outcome.Action == "removed" {
out.Removed++
}
}
// Read back. A removal that reported success and left the container running would leave two
// control planes consuming the same broker queues for ever, which is the state this step
// exists to end.
gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name)
if err != nil {
return out, err
}
out.Gone = gone
if !gone {
return out, fmt.Errorf(
"the apply reported the temporary control plane removed and %q is still running.\n"+
"Two control planes are consuming this mesh's broker queues. Neither is wrong and "+
"the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+
"it, and this installer will then agree", temporary.Name, temporary.Name)
}
say(" gone " + temporary.Name)
return out, nil
}
// removeResource takes one resource out of a bundle's text, comments and all.
//
// It walks the `resources` array counting braces, skipping over strings and comments so that a
// `//` inside a connection string is not read as the start of one — the substrate's own bundle
// contains `postgres://…` several times, and a scanner that did not know the difference would
// treat the rest of the line as a comment and lose a brace.
//
// What is removed is the element AND whatever precedes it back to the previous element, which is
// where the comment explaining it lives. A comment that outlives the thing it describes is worse
// than no comment: it is the file telling somebody the machine has a control plane it does not.
func removeResource(bundle []byte, id string) ([]byte, error) {
array := indexOutsideStrings(bundle, `"resources"`)
if array < 0 {
return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
}
open := indexOutsideStrings(bundle[array:], "[")
if open < 0 {
return nil, fmt.Errorf("this bundle's resources are not a list")
}
open += array
depth, from := 0, -1
previous := open
inString, escaped, inLine, inBlock := false, false, false, false
for i := open + 1; i < len(bundle); i++ {
c := bundle[i]
switch {
case escaped:
escaped = false
case inString && c == '\\':
escaped = true
case inString:
if c == '"' {
inString = false
}
case inLine:
if c == '\n' {
inLine = false
}
case inBlock:
if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' {
inBlock, i = false, i+1
}
case c == '"':
inString = true
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/':
inLine, i = true, i+1
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*':
inBlock, i = true, i+1
case c == '{':
if depth == 0 {
from = i
}
depth++
case c == '}':
depth--
if depth != 0 {
break
}
if isResource(bundle[from:i+1], id) {
return cut(bundle, previous, from, i+1), nil
}
previous = i + 1
from = -1
case c == ']' && depth == 0:
return nil, fmt.Errorf(
"this bundle declares no %q, so there is nothing to take out of it", id)
}
}
return nil, fmt.Errorf("this bundle's resources list does not end")
}
// isResource reports whether one resource's text is the one wanted.
//
// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and
// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not.
func isResource(resource []byte, id string) bool {
var tight []byte
for _, c := range resource {
if c != ' ' && c != '\t' && c != '\n' && c != '\r' {
tight = append(tight, c)
}
}
return bytes.Contains(tight, []byte(`"id":"`+id+`"`))
}
// cut removes an element and what leads up to it, leaving the list valid.
//
// Whether the comma before or the comma after goes depends on where the element sits: an element
// with something before it takes the comma that joined them, and the first element takes the one
// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse —
// caught by the re-parse either way, and better not produced.
func cut(bundle []byte, previous, from, to int) []byte {
start := from
for i := previous; i < from; i++ {
if bundle[i] == ',' {
start = i
break
}
}
end := to
if start == from {
// Nothing before it, so the comma that follows is the one that would be left dangling.
for i := to; i < len(bundle); i++ {
if bundle[i] == ',' {
end = i + 1
break
}
if bundle[i] == ']' {
break
}
}
}
out := make([]byte, 0, len(bundle))
out = append(out, bundle[:start]...)
return append(out, bundle[end:]...)
}
// indexOutsideStrings finds a fragment that is not inside a JSON string.
func indexOutsideStrings(haystack []byte, needle string) int {
inString, escaped := false, false
for i := 0; i < len(haystack); i++ {
switch {
case escaped:
escaped = false
continue
case haystack[i] == '\\' && inString:
escaped = true
continue
case haystack[i] == '"':
// The needle may itself start with a quote, so the match is tried before the quote is
// consumed.
if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) {
return i
}
inString = !inString
continue
case inString:
continue
}
if bytes.HasPrefix(haystack[i:], []byte(needle)) {
return i
}
}
return -1
}
// isGone waits for a container to stop existing.
//
// Waited for rather than asked once, because a container being removed is a container that is
// stopping first, and a runtime answers about it until it has finished.
func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) {
deadline := time.Now().Add(wait)
for {
if _, err := containerRunning(ctx, run, probe, name); err != nil {
// The runtime does not know it. That is the answer being waited for.
return true, nil
}
if time.Now().After(deadline) {
return false, nil
}
select {
case <-ctx.Done():
return false, ctx.Err()
case <-time.After(answerEvery):
}
}
}
internal/bootstrap/retire_test.go
+165
View File
@@ -0,0 +1,165 @@
package bootstrap
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// Retirement is destruction by omission, which is the host's ordinary behaviour: it owns what it
// applied and removes what it owns and is no longer declared. These tests defend the bundle
// surgery that expresses it, because a bundle that came out of it unparseable would be found by
// the apply — after the file on the machine had already been replaced.
func produced(t *testing.T) []byte {
t.Helper()
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
return out.Bundle
}
// The temporary control plane leaves the bundle, everything else stays, and what is left parses.
func TestTheTemporaryControlPlaneLeavesTheBundleAndNothingElseDoes(t *testing.T) {
before, err := declaration.ParseFileTrusted(produced(t))
if err != nil {
t.Fatal(err)
}
shorter, err := removeResource(produced(t), ControlPlaneID)
if err != nil {
t.Fatal(err)
}
after, err := declaration.ParseFileTrusted(shorter)
if err != nil {
t.Fatalf("the bundle without the control plane does not parse: %v\n%s", err, shorter)
}
if len(after.Resources) != len(before.Resources)-1 {
t.Fatalf("the bundle went from %d resources to %d, and one was removed",
len(before.Resources), len(after.Resources))
}
if _, err := controlPlaneIn(after); err == nil {
t.Error("the bundle still declares a control plane")
}
// The store and the broker are still exactly what they were. A retirement that took the
// substrate with it would leave the machine with a module and nothing under it.
for id, name := range containerNames(before) {
if id == ControlPlaneID {
continue
}
if containerNames(after)[id] != name {
t.Errorf("%s was lost or renamed by the retirement", id)
}
}
}
// **A `//` inside a string is not a comment.** The substrate's own bundle carries
// `postgres://…` several times, and a scanner that read the rest of those lines as a comment
// would lose braces and cut the wrong thing out — silently, because what it produced would still
// look like a file.
func TestASchemeInsideAStringIsNotReadAsAComment(t *testing.T) {
shorter, err := removeResource(produced(t), ControlPlaneID)
if err != nil {
t.Fatal(err)
}
after, err := declaration.ParseFileTrusted(shorter)
if err != nil {
t.Fatal(err)
}
// The migration action, which is the resource holding the most `://` of anything here, is
// still whole.
found := false
for _, r := range after.Resources {
if r.Identity() == "context-schemas" {
found = true
}
}
if !found {
t.Error("the resource full of connection strings did not survive the removal")
}
}
// Removing the FIRST element takes the comma after it rather than the comma before it, because
// there is no comma before it. Getting this wrong produces a leading comma, which is JSON nothing
// parses — and the file would already have been written.
func TestRemovingTheFirstResourceLeavesAValidList(t *testing.T) {
shorter, err := removeResource(produced(t), "container-runtime")
if err != nil {
t.Fatal(err)
}
after, err := declaration.ParseFileTrusted(shorter)
if err != nil {
t.Fatalf("removing the first resource broke the bundle: %v\n%s", err, shorter)
}
for _, r := range after.Resources {
if r.Identity() == "container-runtime" {
t.Error("the first resource is still there")
}
}
}
// A bundle that declares no such resource is refused rather than silently returned unchanged. A
// removal that removed nothing and reported success would leave the apply below with nothing to
// do and the installer claiming a pivot it did not finish.
func TestRemovingSomethingThatIsNotThereIsRefused(t *testing.T) {
if _, err := removeResource(produced(t), "nothing-of-the-sort"); err == nil {
t.Fatal("a bundle was reported to have had a resource removed that it never declared")
}
}
// A re-run after the retirement finds a bundle with no control plane in it and says so, rather
// than failing. This is the idempotence of the last step, and it is the one a person is most
// likely to exercise: the pivot ends here, so a re-run to check ends here too.
func TestRetiringABundleThatAlreadyHasNoControlPlaneIsAlreadyDone(t *testing.T) {
shorter, err := removeResource(produced(t), ControlPlaneID)
if err != nil {
t.Fatal(err)
}
var said []string
out, err := RetireTheTemporaryControlPlane(context.Background(), Options{},
nil, shorter, nil, func(line string) { said = append(said, line) })
if err != nil {
t.Fatal(err)
}
if !out.Already {
t.Error("a bundle with no control plane in it was not reported as already retired")
}
if !strings.Contains(strings.Join(said, "\n"), "already dropped") {
t.Errorf("the run does not say it was already done: %v", said)
}
}
// A container the runtime still knows about after the apply is a pivot that did not finish. Two
// control planes on one mesh are both correct and neither is wrong — but the temporary one was
// supposed to go, and saying it went when it did not is the fault this project keeps naming.
func TestAContainerStillThereAfterRemovalIsNotGone(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
stillThere := &asked{answer: func(_ string, _ []string) (string, error) {
return "true running\n", nil
}}
gone, err := isGone(context.Background(), stillThere.run, time.Second, 0, "temp-mesh-control")
if err != nil {
t.Fatal(err)
}
if gone {
t.Error("a container the runtime still describes was reported gone")
}
removed := &asked{answer: func(_ string, _ []string) (string, error) {
return "", errors.New("No such object: temp-mesh-control")
}}
gone, err = isGone(context.Background(), removed.run, time.Second, 0, "temp-mesh-control")
if err != nil {
t.Fatal(err)
}
if !gone {
t.Error("a container the runtime does not know about was not reported gone")
}
}
internal/bootstrap/rewrite.go
+336
View File
@@ -0,0 +1,336 @@
package bootstrap
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/novox/mesh-host/internal/declaration"
)
// ControlPlaneID is the resource the installer replaces the image of.
//
// A resource id rather than a container name or a guess at the image, because the id is the one
// thing a declaration promises is stable — it is what lets the store say *this is the same
// resource I applied last time* (`internal/declaration`, Resource.Identity). A bundle that does not
// name one is refused rather than applied without a control plane, which would raise a store and a
// broker and no mesh.
const ControlPlaneID = "control-plane"
// TempPrefix is what the substrate's control plane is renamed with.
//
// **This is the whole of how a carried resource becomes a declared one.** The substrate raises a
// control plane and a module later declares one, and for a moment both exist — which looked like a
// handover problem needing a way for the host to stop owning something without destroying it. It is
// not one. The temporary control plane is called `temp-mesh-control` and the permanent one is
// called `mesh-control`: two containers, two owners, nothing shared and nothing to hand over. At
// the end the temporary one is dropped from the bundle and the host removes it, which is exactly
// what should happen to something named "temp" (novox/hq ADR 0067).
//
// The name is also the audit. After the pivot, a machine running `mesh-control` and not
// `temp-mesh-control` has completed it; one running both stopped in the middle; one running only
// the temp has not started. That is readable from `docker ps` by somebody who knows nothing else.
const TempPrefix = "temp-"
// ControlPlaneModule is the module the permanent control plane is installed as, and the name its
// container takes — the name the substrate's own control plane gives up here so that it can.
//
// Declared beside the rename rather than beside the step that uses it, because this is where the
// two names are decided together and where the reason for both of them is written down.
const ControlPlaneModule = "mesh-control"
// brokerAddressVar is what a token tells an enrolling node to dial.
//
// Not rewritten here — see the note in Rewrite — but reported, because it is the field most likely
// to be wrong on a machine that is not the one the template was written for, and it is wrong in a
// way nothing notices until a second node tries to join.
const brokerAddressVar = "MESH_BROKER_ADDRESS"
// Rewritten is the bundle this machine will apply, and what was done to produce it.
type Rewritten struct {
// Bundle is the produced file's bytes — the template with one image reference replaced,
// comments and all.
Bundle []byte
// Declaration is that bundle, parsed. Carried so the apply and the verify are talking about
// the same document rather than each re-reading the file and hoping.
Declaration *declaration.Declaration
// Was is the image the template named the control plane by; Now is the one it names it by.
Was string
Now string
// Places is how many times that reference appeared, and therefore how many were replaced.
Places int
// Changed is false when the template already named this image — a re-run on a bundle this
// installer produced earlier.
Changed bool
// WasCalled is what the template called the control plane's container; TempName is what the
// produced bundle calls it. Renamed is false when the template already used the temporary name.
WasCalled string
TempName string
Renamed bool
// Kept is every other container image, unchanged, as "<name> <image>". Reported rather than
// assumed: "postgres was left alone" is a claim, and this is the evidence for it.
Kept []string
// Resources is how many things the produced bundle asks for.
Resources int
// BrokerAddress is what the control plane will tell enrolling nodes to dial, or empty.
BrokerAddress string
}
// Rewrite produces the bundle this machine will apply from the template it was given.
//
// **Two substitutions, and both are textual.** The control plane's image becomes the id of the image
// this machine now holds, and its container is renamed `temp-mesh-control`; nothing else changes.
// The rename is what makes the pivot expressible at all — see TempPrefix. Textual rather than
// parse-and-re-serialise because
// the produced file has to be *read* — a person getting a machine working must be able to open it,
// see the substrate they recognise, and see exactly one thing different. Re-serialising a parsed
// declaration would drop every comment in the template, and those comments are where the reasons
// live.
//
// **Every place that reference appears, not only the container.** The bundle names the control
// plane's image twice: once as the container that runs `serve`, and once inside the action that
// runs `migrate` to create the contexts' schemas. Replacing only the container would leave the
// migration pointing at an image no registry serves, and the apply would fail in the middle —
// after the store is up, before the broker. They are one image and they move together.
//
// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references
// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is
// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other
// container's image is compared against what it was.
//
// **What this deliberately does NOT rewrite:** MESH_BROKER_ADDRESS, the endpoint every enrolment
// token will carry. It differs per machine and it is silently fatal when wrong — a node enrols
// against a dead address and nothing complains until it fails to come back. It belongs to the
// enrolment stage, which is not built yet, so this reports it loudly and leaves it alone rather
// than guessing an address for a machine it has not been told about.
func Rewrite(template []byte, imageID string) (Rewritten, error) {
if !isImageID(imageID) {
return Rewritten{}, fmt.Errorf(
"%q is not an image id. The control plane is named by the digest of its own "+
"configuration — sha256: and sixty-four hex characters — because nothing serves "+
"it and there is no manifest digest to use instead", imageID)
}
before, err := declaration.ParseFileTrusted(template)
if err != nil {
return Rewritten{}, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
control, err := controlPlaneIn(before)
if err != nil {
return Rewritten{}, err
}
out := Rewritten{Was: control.Image, Now: imageID, BrokerAddress: control.Env[brokerAddressVar]}
occurrences := bytes.Count(template, []byte(control.Image))
if occurrences == 0 {
// The parser found the image and the bytes do not contain it, which means the two are
// reading different things. Refused rather than replaced-zero-times-and-reported-success.
return Rewritten{}, fmt.Errorf(
"the control plane's image is %q according to the parsed template, and that text is "+
"not in the file. Nothing was rewritten", control.Image)
}
out.Places = occurrences
switch {
case control.Image == imageID:
// Already this image. The idempotent case, and the one that happens whenever somebody
// re-runs the installer against a bundle it produced earlier.
out.Bundle = template
default:
out.Bundle = bytes.ReplaceAll(template, []byte(control.Image), []byte(imageID))
out.Changed = true
}
// And the container is renamed, for the reason TempPrefix records. Done here rather than
// anywhere later because the bundle is the only place the name is decided: the apply creates
// the container from it, the verify asks that container questions, and the retirement takes
// this same resource back out. One name, one place, read by everything.
out.WasCalled, out.TempName = control.Name, TempPrefix+control.Name
if strings.HasPrefix(control.Name, TempPrefix) {
out.TempName = control.Name
}
renamed, err := renameContainer(out.Bundle, control.Name, out.TempName)
if err != nil {
return Rewritten{}, err
}
out.Bundle, out.Renamed = renamed, out.TempName != control.Name
// Read back, on the bytes that will actually be applied. Everything above is an intention
// until the produced file is parsed and asked what it says.
after, err := declaration.ParseFileTrusted(out.Bundle)
if err != nil {
return Rewritten{}, fmt.Errorf(
"the bundle this produced is not a declaration, so the substitution broke it: %w", err)
}
out.Declaration, out.Resources = after, len(after.Resources)
produced, err := controlPlaneIn(after)
if err != nil {
return Rewritten{}, err
}
if produced.Image != imageID {
return Rewritten{}, fmt.Errorf(
"the produced bundle still names the control plane %q, not %q", produced.Image, imageID)
}
if produced.Name != out.TempName {
return Rewritten{}, fmt.Errorf(
"the produced bundle still calls the control plane's container %q, not %q. The "+
"permanent one is a module and takes the plain name, so a substrate that kept it "+
"would put two owners on one container", produced.Name, out.TempName)
}
// And nothing else moved. A substitution on text can in principle catch more than it was
// aimed at, and "postgres was left exactly as it was" is the claim this checks rather than
// asserts. Both the image AND the name, because there are now two substitutions.
wasImage, wasName := containerImages(before), containerNames(before)
for id, image := range containerImages(after) {
if id == ControlPlaneID {
continue
}
if wasImage[id] != image {
return Rewritten{}, fmt.Errorf(
"rewriting the control plane's image also changed %q, from %q to %q. Only the "+
"mesh's own image may move; everything else is somebody else's image at "+
"somebody else's registry", id, wasImage[id], image)
}
out.Kept = append(out.Kept, fmt.Sprintf("%s %s", id, image))
}
for id, name := range containerNames(after) {
if id == ControlPlaneID || wasName[id] == name {
continue
}
return Rewritten{}, fmt.Errorf(
"renaming the control plane's container also renamed %q, from %q to %q. Only the "+
"control plane moves out of the way; every other container keeps the name the "+
"substrate gave it", id, wasName[id], name)
}
sortStrings(out.Kept)
return out, nil
}
// renameContainer changes one container's name in the bundle's text.
//
// **The quoted name, not the bare word.** `mesh-control` also appears inside the image reference
// the template carries (`…/mesh-control@sha256:…`) and could appear inside a command line; a bare
// substitution would catch those too. What is wanted is a JSON string that IS the name, so the
// quotes are part of what is matched — `"mesh-control"` matches the container's `name` and an
// action's `in`, which are exactly the places the name means the container, and nothing else.
//
// It refuses when the text does not contain what the parse says is there, for the same reason the
// image substitution does: the two would then be reading different things, and a rename that
// replaced nothing and reported success would leave the module and the substrate fighting over one
// container three steps later.
func renameContainer(bundle []byte, from, to string) ([]byte, error) {
if from == to {
return bundle, nil
}
quoted := []byte(`"` + from + `"`)
if bytes.Count(bundle, quoted) == 0 {
return nil, fmt.Errorf(
"the control plane's container is called %q according to the parsed template, and %s "+
"is not in the file. Nothing was renamed, and the substrate would raise a "+
"container the module also wants", from, quoted)
}
return bytes.ReplaceAll(bundle, quoted, []byte(`"`+to+`"`)), nil
}
// controlPlaneIn finds the container this installer replaces the image of.
func controlPlaneIn(d *declaration.Declaration) (*declaration.Container, error) {
for _, r := range d.Resources {
if r.Identity() != ControlPlaneID {
continue
}
container, ok := r.(*declaration.Container)
if !ok {
return nil, fmt.Errorf(
"this bundle's %q is a %s, and the control plane has to be a container for its "+
"image to be named. Nothing was rewritten", ControlPlaneID, r.Kind())
}
return container, nil
}
return nil, fmt.Errorf(
"this bundle names no %q, so there is no control plane to give this machine's image to. "+
"A substrate without one raises a store and a broker and no mesh. It declares: %s",
ControlPlaneID, strings.Join(identities(d), ", "))
}
func containerImages(d *declaration.Declaration) map[string]string {
images := map[string]string{}
for _, r := range d.Resources {
if container, ok := r.(*declaration.Container); ok {
images[container.ID] = container.Image
}
}
return images
}
func containerNames(d *declaration.Declaration) map[string]string {
names := map[string]string{}
for _, r := range d.Resources {
if container, ok := r.(*declaration.Container); ok {
names[container.ID] = container.Name
}
}
return names
}
func identities(d *declaration.Declaration) []string {
var ids []string
for _, r := range d.Resources {
ids = append(ids, r.Identity())
}
return ids
}
// isImageID is the same shape `internal/declaration` accepts for an image the machine holds. Asked
// here as well so the refusal names the installer's own mistake, rather than surfacing as a
// declaration refusal about a bundle this program wrote.
func isImageID(s string) bool {
const prefix = "sha256:"
if !strings.HasPrefix(s, prefix) || len(s) != len(prefix)+64 {
return false
}
for _, c := range s[len(prefix):] {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
func sortStrings(values []string) {
for i := 1; i < len(values); i++ {
for j := i; j > 0 && values[j] < values[j-1]; j-- {
values[j], values[j-1] = values[j-1], values[j]
}
}
}
// writeBundleFile puts the produced bundle where a person can read it, creating the directory it
// lives in.
//
// 0644, and that is deliberate: this file names an image and describes a substrate, and it holds
// the bootstrap credentials the template happens to carry — which are the same ones anybody can
// read in the template itself. It is meant to be read. What must not be world-readable is the
// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`).
func writeBundleFile(path string, content []byte) error {
if dir := filepath.Dir(path); dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err)
}
}
if err := os.WriteFile(path, content, 0o644); err != nil {
return fmt.Errorf(
"cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+
"applying something nobody can read afterwards is how a machine becomes a mystery",
path, err)
}
return nil
}
internal/bootstrap/rewrite_test.go
+321
View File
@@ -0,0 +1,321 @@
package bootstrap
import (
"os"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
)
// Each test names the decision it defends (novox/hq ADR 0017).
const (
held = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
otherHeld = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
)
// theRealBundle is this repository's own substrate example, used rather than a fixture.
//
// A fixture would agree with whatever this code does. The example is what an installer is actually
// pointed at, it names the control plane twice, and it is the file that changes when the substrate
// changes — so a rewrite that stops working on it is a rewrite that has stopped working.
func theRealBundle(t *testing.T) []byte {
t.Helper()
raw, err := os.ReadFile("../../examples/substrate-first-node.lock")
if err != nil {
t.Fatalf("reading the substrate example: %v", err)
}
return raw
}
func TestTheControlPlaneIsNamedByTheImageThisMachineHolds(t *testing.T) {
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
if !out.Changed {
t.Error("the rewrite reported nothing changed, and the template named a registry image")
}
control, err := controlPlaneIn(out.Declaration)
if err != nil {
t.Fatal(err)
}
if control.Image != held {
t.Errorf("the control plane is %q, want %q", control.Image, held)
}
}
// **Every place the bundle names that image, not only the container.**
//
// The substrate names the control plane's image twice: the container that runs `serve`, and the
// action that runs `migrate` to create the contexts' schemas. Rewriting only the container leaves
// the migration pointing at an image no registry serves, and the apply dies in the middle — after
// the store is up and before the broker. This is the test that would have caught that.
func TestEveryPlaceTheBundleNamesTheControlPlaneIsRewritten(t *testing.T) {
template := theRealBundle(t)
out, err := Rewrite(template, held)
if err != nil {
t.Fatal(err)
}
if out.Places < 2 {
t.Fatalf("the control plane's image was found in %d place(s); the substrate names it in "+
"the container AND in the migration action", out.Places)
}
if remaining := strings.Count(string(out.Bundle), out.Was); remaining != 0 {
t.Errorf("the produced bundle still names %q in %d place(s)", out.Was, remaining)
}
if got := strings.Count(string(out.Bundle), held); got != out.Places {
t.Errorf("the produced bundle names the held image %d time(s), and %d were replaced",
got, out.Places)
}
}
// Third-party images are somebody else's, at somebody else's registry, and the installer has no
// business touching them (novox/hq ADR 0006).
func TestPostgresAndTheBrokerAreLeftExactlyAsTheyWere(t *testing.T) {
template := theRealBundle(t)
out, err := Rewrite(template, held)
if err != nil {
t.Fatal(err)
}
produced := containerImages(out.Declaration)
for _, id := range []string{"store", "broker"} {
image, named := produced[id]
if !named {
t.Fatalf("the substrate example no longer declares a %q container", id)
}
// Compared against the template's own text rather than against an expectation written
// here: what is being defended is "unchanged", and the template is the only thing that
// knows what it said.
if !strings.Contains(string(template), `"image": "`+image+`"`) {
t.Errorf("%s is now %q, which the template does not say", id, image)
}
if strings.HasPrefix(image, "sha256:") {
t.Errorf("%s was rewritten to an image this machine holds, and nothing holds it", id)
}
}
// And the claim in the report is the same claim, so a person reading it is reading evidence.
if len(out.Kept) != 2 {
t.Errorf("the rewrite reports %d untouched image(s): %v", len(out.Kept), out.Kept)
}
}
// A bundle with no control plane raises a store and a broker and no mesh. Refused, because
// applying it would succeed and leave a machine that looks bootstrapped.
func TestABundleThatNamesNoControlPlaneIsRefused(t *testing.T) {
template := []byte(`{"declaration":1,"resources":[
{"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` +
strings.Repeat("7", 64) + `"}
]}`)
_, err := Rewrite(template, held)
if err == nil {
t.Fatal("a bundle with no control plane was rewritten and would have been applied")
}
// The refusal has to be actionable: it says what the bundle DID declare, so somebody can see
// they pointed it at the wrong file or misspelled the id.
if !strings.Contains(err.Error(), ControlPlaneID) || !strings.Contains(err.Error(), "store") {
t.Errorf("the refusal names neither what was wanted nor what was there: %v", err)
}
}
func TestAControlPlaneThatIsNotAContainerIsRefused(t *testing.T) {
template := []byte(`{"declaration":1,"resources":[
{"id":"control-plane","type":"package","package":"mesh-control"}
]}`)
if _, err := Rewrite(template, held); err == nil {
t.Fatal("a control plane declared as a package was accepted, and a package has no image")
}
}
// Idempotence. This program is run over and over while somebody gets a machine working, and the
// second run must be able to say the bundle already names this image rather than reporting a
// rewrite it did not perform.
func TestRewritingABundleThatAlreadyNamesTheImageChangesNothing(t *testing.T) {
first, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
second, err := Rewrite(first.Bundle, held)
if err != nil {
t.Fatal(err)
}
if second.Changed {
t.Error("re-running the rewrite reported a change, and the image was already the one held")
}
if string(second.Bundle) != string(first.Bundle) {
t.Error("re-running the rewrite produced different bytes")
}
if second.Was != held {
t.Errorf("the second run reports it replaced %q; it replaced nothing", second.Was)
}
}
// A DIFFERENT image, though, must move — the ordinary case of a new control plane being installed
// over an old one. "Already correct" must not be the same code path as "already ran".
func TestANewImageReplacesAnOlderHeldOne(t *testing.T) {
first, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
second, err := Rewrite(first.Bundle, otherHeld)
if err != nil {
t.Fatal(err)
}
if !second.Changed || second.Was != held || second.Now != otherHeld {
t.Errorf("a new image did not replace the old one: changed=%v was=%q now=%q",
second.Changed, second.Was, second.Now)
}
}
// The produced bundle is meant to be READ. Re-serialising a parsed declaration would drop every
// comment in the template, and the substrate example is mostly comments — each one recording why a
// resource is the way it is, several of them paid for in the lab.
func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) {
template := theRealBundle(t)
out, err := Rewrite(template, held)
if err != nil {
t.Fatal(err)
}
const remembered = "NAMED VOLUME"
if !strings.Contains(string(out.Bundle), remembered) {
t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered)
}
}
// The installer must refuse its own bad input in its own words, rather than writing a bundle and
// letting the host refuse a declaration somebody did not write.
func TestSomethingThatIsNotAnImageIdIsRefused(t *testing.T) {
for _, bad := range []string{
"",
"mesh-control:latest",
"sha256:abc",
"sha256:" + strings.Repeat("1", 63),
"sha256:" + strings.Repeat("g", 64),
"mesh-control@sha256:" + strings.Repeat("1", 64),
} {
if _, err := Rewrite(theRealBundle(t), bad); err == nil {
t.Errorf("image id %q was accepted", bad)
}
}
}
// The address every enrolment token will carry is reported and never invented. It differs per
// machine and it is silently fatal when wrong: a node enrols against a dead address and nothing
// says so until it fails to come back.
func TestTheAddressNodesWillDialIsReportedAndNotRewritten(t *testing.T) {
template := theRealBundle(t)
out, err := Rewrite(template, held)
if err != nil {
t.Fatal(err)
}
if out.BrokerAddress == "" {
t.Fatal("the substrate example no longer says what address enrolling nodes will dial")
}
if !strings.Contains(string(out.Bundle), out.BrokerAddress) {
t.Errorf("the produced bundle no longer carries %q — it was rewritten, and nothing here "+
"knows this machine's address", out.BrokerAddress)
}
}
// ---------------------------------------------------------------------------------------------
// The rename, which is what makes genesis a pivot rather than a handover (novox/hq ADR 0067).
// ---------------------------------------------------------------------------------------------
// **This is the test that dissolves the blocker.** The substrate raises a control plane and a
// module later declares one; if both are called `mesh-control` then for one moment two owners hold
// one container, and the host — which tracks what it owns — has no way to stop owning something
// without destroying it. Nothing here invents such a mechanism. The substrate's container is
// called `temp-mesh-control` instead, and there are simply two containers.
func TestTheSubstratesControlPlaneMovesOutOfTheModulesWay(t *testing.T) {
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
if !out.Renamed {
t.Error("the rewrite reported nothing renamed, and the template named it mesh-control")
}
if out.TempName != "temp-mesh-control" {
t.Errorf("the substrate's control plane is called %q", out.TempName)
}
control, err := controlPlaneIn(out.Declaration)
if err != nil {
t.Fatal(err)
}
if control.Name != out.TempName {
t.Errorf("the produced bundle calls it %q, want %q", control.Name, out.TempName)
}
// And the plain name is free, which is the whole point: it belongs to the module now.
for _, name := range containerNames(out.Declaration) {
if name == ControlPlaneModule {
t.Errorf("the produced bundle still declares a container called %q, which the module "+
"will also declare", ControlPlaneModule)
}
}
}
// The image reference contains the string `mesh-control` too, and it is not a container name. A
// substitution that caught it would produce `…/temp-mesh-control@sha256:…`, which no registry
// serves — and it would be found inside a pull rather than here.
func TestTheImageReferenceIsNotMistakenForTheContainerName(t *testing.T) {
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(out.Bundle), TempPrefix+"mesh-control@") ||
strings.Contains(string(out.Bundle), "/"+TempPrefix+"mesh-control") {
t.Error("the rename reached inside an image reference")
}
}
// Everything else keeps the name the substrate gave it. The store and the broker are containers
// too, and a rename that moved them would leave a machine whose substrate the host cannot find.
func TestRenamingTheControlPlaneLeavesEveryOtherContainerAlone(t *testing.T) {
before, err := declaration.ParseFileTrusted(theRealBundle(t))
if err != nil {
t.Fatal(err)
}
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
was, now := containerNames(before), containerNames(out.Declaration)
for id, name := range was {
if id == ControlPlaneID {
continue
}
if now[id] != name {
t.Errorf("%s was renamed from %q to %q", id, name, now[id])
}
}
}
// A re-run against a bundle this installer produced renames nothing and says so. The installer is
// run over and over while somebody gets a machine working, and a step that could not tell "already
// done" from "just done" makes the second run indistinguishable from the first.
func TestRewritingABundleThisAlreadyProducedRenamesNothing(t *testing.T) {
first, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
second, err := Rewrite(first.Bundle, held)
if err != nil {
t.Fatal(err)
}
if second.Renamed {
t.Error("a bundle already naming temp-mesh-control was renamed again")
}
if second.TempName != first.TempName {
t.Errorf("the second pass calls it %q and the first called it %q",
second.TempName, first.TempName)
}
if string(second.Bundle) != string(first.Bundle) {
t.Error("rewriting a produced bundle changed it")
}
}
internal/bootstrap/talk.go
+119
View File
@@ -0,0 +1,119 @@
package bootstrap
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// controlPlane is the running control plane, asked things.
//
// **Through `docker exec`, not over a network.** The control plane listens on nothing — `serve` is
// a broker consumer, and every administrative verb is a subcommand of the same binary that opens
// the stores directly (mesh-control's own usage). So the way to tell a mesh anything, from the
// machine the mesh is on, is to run its binary inside its own container. That is also what the lab
// does, and having the installer and the lab drive the mesh identically is the point: the lab is
// meant to exercise the installer, not a second procedure that resembles it.
//
// It carries which container, because the whole pivot turns on there being two of them: the
// substrate's `temp-mesh-control` for steps 6 to 9, and the module's `mesh-control` afterwards.
type controlPlane struct {
container string
run Runner
timeout time.Duration
}
// within is the same control plane, asked with a different patience.
//
// A copy rather than a field somebody sets, so a slow command cannot leave every command after it
// slow: the caller that needs the long wait says so on the call.
func (c controlPlane) within(timeout time.Duration) controlPlane {
c.timeout = timeout
return c
}
// tell runs a mesh-control subcommand and gives back what it said.
//
// The failure carries the command AND the output. A mesh-control refusal is a paragraph explaining
// what is wrong — "nothing provides route, wanted by registry" — and an installer that reported
// only "exit status 1" would throw away the one thing a person needs.
func (c controlPlane) tell(ctx context.Context, args ...string) (string, error) {
asking, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
out, err := c.run(asking, "docker", append(
[]string{"exec", c.container, controlPlaneBinary}, args...)...)
if err != nil {
return out, fmt.Errorf("`%s %s` was refused: %w\n%s",
c.container, strings.Join(args, " "), err, indent(strings.TrimSpace(out)))
}
return out, nil
}
// carry puts a file inside the control plane's container.
//
// **Because every command that takes a file reads it from its own filesystem.** `module add
// <file>` and `secret accept --from <file>` open a path, and the process doing the opening is
// inside the container. The installer is not. So the file is copied in first, exactly as the lab
// does it.
//
// The image is `FROM scratch` and has no shell, so nothing inside can move a file, change its mode
// or clean up after itself. What is copied in stays until the container is replaced — which, for
// the temporary control plane, is a container that gets removed at step 10 and takes its contents
// with it.
func (c controlPlane) carry(ctx context.Context, local, remote string) error {
asking, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
if _, err := c.run(asking, "docker", "cp", local, c.container+":"+remote); err != nil {
return fmt.Errorf("cannot put %s into %s at %s: %w", local, c.container, remote, err)
}
return nil
}
// carrying writes bytes to a temporary file on the machine and copies them into the container.
//
// **0644, and that is not carelessness — 0600 would break it.** `docker cp` keeps the ownership and
// mode a file had outside, the control plane's image runs as 65534, and the process that reads
// these files is that one. The lab paid for this exactly once: a 0600 root-owned key copied in
// landed unreadable, `secret accept` failed with `permission denied`, and what depended on it
// crash-looped on material it never received. There is no shell in the image to chown it with.
//
// What goes through here is a module manifest and a store connection string. The connection is the
// same value the produced bundle already holds in the clear — a substrate names its own bootstrap
// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The
// file on the machine is removed at once, and the copy inside the container goes when the
// container does, which for the temporary control plane is step 10.
func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error {
local := filepath.Join(os.TempDir(), name)
if err := os.WriteFile(local, content, 0o644); err != nil {
return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err)
}
defer os.Remove(local)
return c.carry(ctx, local, remote)
}
func indent(s string) string {
if s == "" {
return ""
}
return " " + strings.ReplaceAll(s, "\n", "\n ")
}
// mentions reports whether one of a listing's lines starts with this exact word.
//
// Line-and-word rather than a substring search, because these listings are columns and a
// substring match would find `registry` inside `registry-mirror` and report a module installed
// that is not. Every one of mesh-control's `list` verbs prints the name first on the line.
func mentions(listing, name string) bool {
for _, line := range strings.Split(listing, "\n") {
first, _, _ := strings.Cut(strings.TrimSpace(line), " ")
if first == name {
return true
}
}
return false
}
internal/bootstrap/verify.go
+167
View File
@@ -0,0 +1,167 @@
package bootstrap
import (
"context"
"fmt"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// controlPlaneBinary is where the control plane's program lives in its own image.
//
// A path rather than a shell command, because the image is `FROM scratch` and holds one static
// binary and nothing else — no shell to invoke, nothing to interpret a command line
// (mesh-control's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer
// carries, which is why the path can be written down here.
const controlPlaneBinary = "/mesh-control"
// answerEvery is how often the control plane is asked again while it is starting.
var answerEvery = 2 * time.Second
// Verified is what the substrate was found to be.
type Verified struct {
// Running is every long-running container the bundle declares, confirmed up.
Running []string
// Answered is the control plane's own first words back, so the report shows the reply rather
// than asserting there was one.
Answered string
}
// Verify proves the substrate is up and the control plane replies.
//
// **A container that is up is not a control plane that replies**, and this project has paid for
// that distinction more than once: a runtime reports a container running from the moment the
// process starts, which is before it has opened a database, before it has read its configuration,
// and before it has failed to. So the containers are checked, and then the program inside one of
// them is asked a question and has to answer it.
//
// The question is `status`, and it is chosen rather than convenient: answering it means the
// control plane opened all three of its stores from the environment the bundle gave it. A reply
// therefore proves the image runs, that `network: host` really does reach the store on this
// machine, and that the contexts' schemas migrated — the three things the steps before this were
// for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server.
//
// It waits. A control plane that is not answering yet and a control plane that will never answer
// look identical for the first few seconds, and refusing on the first attempt would make a correct
// bootstrap fail for being observed too early.
func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration,
say func(string)) (Verified, error) {
var out Verified
control, err := controlPlaneIn(d)
if err != nil {
return out, err
}
for _, container := range longRunning(d) {
state, err := containerRunning(ctx, run, probe, container)
if err != nil {
return out, err
}
if !state.running {
return out, fmt.Errorf(
"the container %q is %s, not running.\n"+
"The apply reported success, so it was created — what it did afterwards is "+
"in `docker logs %s`", container, state.status, container)
}
out.Running = append(out.Running, container)
say(" running " + container)
}
answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say)
if err != nil {
return out, err
}
out.Answered = answer
return out, nil
}
func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration,
container string, say func(string)) (string, error) {
deadline := time.Now().Add(wait)
var last error
for {
asking, cancel := context.WithTimeout(ctx, probe)
out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status")
cancel()
answer := strings.TrimSpace(firstLineOf(out))
switch {
case err != nil:
last = err
case answer == "":
// Exit zero and nothing said. Treated as no answer rather than as success: a program
// that returns silence is not one that has been asked anything.
last = fmt.Errorf("it exited without saying anything")
default:
say(" replies " + container + ": " + answer)
return answer, nil
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(answerEvery):
}
}
return "", fmt.Errorf(
"the container %q is running and the control plane in it does not answer, after waiting "+
"%s: %v\n"+
"Running is not replying. `status` opens this mesh's three stores, so what failed is "+
"most likely the store or the schemas rather than the control plane itself — "+
"`docker logs %s` says which", container, wait, last, container)
}
type containerState struct {
running bool
status string
}
func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) {
asking, cancel := context.WithTimeout(ctx, probe)
defer cancel()
// Both facts in one answer, so a container that is not running is reported with what it IS
// rather than with the absence of what it should be.
out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name)
if err != nil {
return containerState{}, fmt.Errorf(
"the container %q is not there at all, and the apply reported it applied: %w", name, err)
}
running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ")
if status == "" {
status = "in a state the runtime did not name"
}
return containerState{running: running == "true", status: status}, nil
}
// longRunning is every container the bundle expects to still be there afterwards.
//
// A run-once step has exited by design and a scheduled step has deliberately never been started
// (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the
// substrate to be something other than what it declared.
func longRunning(d *declaration.Declaration) []string {
var names []string
for _, r := range d.Resources {
container, ok := r.(*declaration.Container)
if !ok || container.RunOnce || container.Schedule != "" {
continue
}
names = append(names, container.Name)
}
return names
}
func firstLineOf(s string) string {
if i := strings.IndexByte(s, '\n'); i >= 0 {
return s[:i]
}
return s
}
internal/bootstrap/verify_test.go
+170
View File
@@ -0,0 +1,170 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
func substrate(t *testing.T) *declaration.Declaration {
t.Helper()
out, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
return out.Declaration
}
// **A container that is up is not a control plane that replies**, and this project has paid for
// that distinction more than once. A runtime reports a container running from the moment its
// process starts — before it has opened a database, and before it has failed to.
func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
runtime := &asked{answer: func(_ string, args []string) (string, error) {
switch args[0] {
case "inspect":
return "true running\n", nil
case "exec":
// Up, and saying nothing. The program inside is not answering.
return "", errors.New("exit status 1")
}
return "", fmt.Errorf("unexpected command: %v", args)
}}
_, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, 0, func(string) {})
if err == nil {
t.Fatal("every container was running, nothing answered, and the substrate was reported up")
}
for _, wanted := range []string{"mesh-control", "Running is not replying", "docker logs"} {
if !strings.Contains(err.Error(), wanted) {
t.Errorf("the failure does not mention %q:\n%v", wanted, err)
}
}
}
// Exit zero and silence is not an answer either. A program that returns nothing has not been asked
// anything, and treating it as success is the same fault one level down.
func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return "true running\n", nil
}
return " \n", nil
}}
if _, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, 0, func(string) {}); err == nil {
t.Fatal("a control plane that exited zero without saying anything was accepted")
}
}
// The substrate answering is the whole point, and what it said is reported rather than asserted.
func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return "true running\n", nil
}
return "1 node, 0 waiting\n", nil
}}
verified, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, 0, func(string) {})
if err != nil {
t.Fatal(err)
}
// Three long-running containers: the store, the broker and the TEMPORARY control plane. The
// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
// fault. The name is `temp-mesh-control` because the permanent one is a module and takes the
// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
want := []string{"mesh-store", "mesh-broker", "temp-mesh-control"}
if len(verified.Running) != len(want) {
t.Fatalf("confirmed %v running, want %v", verified.Running, want)
}
for i := range want {
if verified.Running[i] != want[i] {
t.Errorf("confirmed %v running, want %v", verified.Running, want)
}
}
if verified.Answered != "1 node, 0 waiting" {
t.Errorf("the control plane's reply is reported as %q", verified.Answered)
}
}
// A control plane that is still opening its stores is waited for, not refused. Refusing on the
// first attempt would make a correct bootstrap fail for being observed too early.
func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
attempts := 0
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return "true running\n", nil
}
attempts++
if attempts < 3 {
return "", errors.New("exit status 1")
}
return "1 node\n", nil
}}
if _, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, time.Second, func(string) {}); err != nil {
t.Fatalf("a control plane that answered on the third ask was refused: %v", err)
}
}
// A container that exited is named with what it IS, so somebody can go and read its logs rather
// than being told only that something is not what it should be.
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" {
return "false exited\n", nil
}
if args[0] == "inspect" {
return "true running\n", nil
}
return "", fmt.Errorf("unexpected command: %v", args)
}}
_, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, 0, func(string) {})
if err == nil {
t.Fatal("a container that had exited was reported as part of a running substrate")
}
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") {
t.Errorf("the failure does not say which container is in what state: %v", err)
}
}
// The control plane is asked by running the binary in its own image directly, because the image is
// `FROM scratch` and has no shell for a command line to be interpreted by.
func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "inspect" {
return "true running\n", nil
}
return "1 node\n", nil
}}
if _, err := Verify(context.Background(), substrate(t), runtime.run,
time.Second, 0, func(string) {}); err != nil {
t.Fatal(err)
}
if !runtime.ran("docker exec " + TempPrefix + "mesh-control " + controlPlaneBinary + " status") {
t.Errorf("the control plane was never asked anything: %v", runtime.commands)
}
}
internal/declaration/declaration.go
+24 -3
View File
@@ -874,20 +874,41 @@ func checkMode(where, mode string) []string {
return nil
}
// checkImage insists on a digest.
// checkImage insists on content, not on a name.
//
// A tag moves and a digest does not. The bundle's whole claim is that what it names is exact
// (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever
// that tag points at on the day the host happens to run.
//
// **Two forms say something exact, and only one of them needs a registry.** `name@sha256:…` is a
// manifest digest, which a registry assigns on push. A bare `sha256:…` is an image the machine
// already holds, addressed by the digest of its own configuration — equally immutable, equally
// unforgeable, and requiring nothing to have served it.
//
// That second form is what a first machine needs. The mesh's own control plane exists in no public
// registry and never will: it is built from source, and until this mesh has a registry of its own
// there is nowhere to push it to and therefore no manifest digest to name it by. Insisting on one
// would mean a registry has to exist before the thing that lets a mesh have a registry can start —
// which is not a pin, it is a dependency the rule accidentally created. A machine that built an
// image, or was handed one, can name it by what it is.
func checkImage(where, image string) []string {
if image == "" {
return []string{where + ": a container needs an image"}
}
// An image this machine holds, named by the digest of its own configuration.
if strings.HasPrefix(image, "sha256:") {
if len(image) != len("sha256:")+64 {
return []string{fmt.Sprintf(
"%s: image id %q is not a sha256 digest", where, image)}
}
return nil
}
name, digest, found := strings.Cut(image, "@")
if !found || name == "" {
return []string{fmt.Sprintf(
"%s: image %q is not pinned. Write it as name@sha256:... — a tag moves, and a "+
"bundle that pinned a tag would not be pinned", where, image)}
"%s: image %q is not pinned. Write it as name@sha256:… — or as sha256:… for an image "+
"this machine already holds. A tag moves, and a bundle that pinned a tag would "+
"not be pinned", where, image)}
}
if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 {
return []string{fmt.Sprintf(
internal/declaration/declaration_test.go
+26
View File
@@ -219,6 +219,32 @@ func TestAnImageMustBePinnedByDigest(t *testing.T) {
}
}
// An image the machine already holds, addressed by the digest of its own configuration.
//
// **The form a first machine needs.** A manifest digest is assigned by a registry on push, so
// insisting on one means a registry has to exist before the thing that lets a mesh have a registry
// can start. The mesh's own control plane is built from source and lives in no public registry; a
// machine that built it, or was handed it, names it by what it is — a content address, not a name,
// and immutable in exactly the way the rule asks for.
func TestAnImageTheMachineHoldsIsNamedByItsOwnDigest(t *testing.T) {
held := "sha256:" + strings.Repeat("b", 64)
if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[
{"id":"control","type":"container","name":"mesh-control","image":"` + held + `"}
]}`)); err != nil {
t.Errorf("an image named by its own digest was refused: %v", err)
}
// Still a digest, though. A truncated one names several images, and which one ran would be
// whichever the runtime happened to match first.
for _, bad := range []string{"sha256:abc", "sha256:", "sha256:" + strings.Repeat("b", 63)} {
if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[
{"id":"control","type":"container","name":"mesh-control","image":"` + bad + `"}
]}`)); err == nil {
t.Errorf("image id %q was accepted and is not a digest", bad)
}
}
}
func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) {
// The field-set check must cover the types added last, not only the three it was written
// for. A package that carries a `content` is a control plane believing it asked for
internal/image/control-plane.tar
+8
View File
@@ -0,0 +1,8 @@
This is not a saved image. It is the placeholder that keeps this repository buildable.
A release build replaces this file with the output of `docker save` and puts it back afterwards:
make bootstrap IMAGE=mesh-control:<version>
An installer built with this file present carries no control plane, and says so in preflight
rather than getting a machine part-way to being a mesh and stopping.
internal/image/image.go
+190
View File
@@ -0,0 +1,190 @@
// Package image is the control plane's image, carried inside the installer.
//
// **Carried rather than built, and that is not an optimisation.** The mesh's forge runs on the
// mesh. A bootstrap that needed the control plane's source in order to build it would need a
// forge to fetch that source from, and the forge is one of the things the mesh raises — so the
// mesh would be required in order to raise the mesh. Embedding the image breaks that cycle the
// same way `internal/bundle` breaks it for the declaration: the installer arrives holding
// everything a bare machine has to be given, and a bare machine is given one file
// (novox/hq ADR 0005).
//
// It is also the rule the rest of tier 0 already follows. Nobody compiles `mesh-host` on the
// machine it will run on; the binary is put there. The image it raises arrives the same way.
//
// What is embedded is the output of `docker save` — a tar holding the image's config, its layers
// and a `manifest.json` naming them. The control plane's image is `FROM scratch` with one static
// binary in it (novox/hq ADR 0006), so this is tens of megabytes rather than hundreds.
package image
import (
"archive/tar"
"bytes"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"strings"
)
// The saved image, replaced at release time by `make bootstrap`.
//
// What is committed here is a placeholder, for the same reason `internal/bundle` commits locks
// that are only comments: `go:embed` refuses to compile against a file that is not there, so a
// checkout with nothing embedded would not build at all — and someone reading this repository or
// running `go test ./...` would meet a compile error instead of a program. The placeholder keeps
// the tree buildable and makes the absence a thing the installer *says*, at the earliest moment
// it can, rather than a thing a compiler says to the wrong person.
//
// It is small and it is committed. A saved image is not, and `make bootstrap` puts one here for
// the length of one build and then puts the placeholder back — exactly what `make host` does with
// the bundle it embeds.
//
//go:embed control-plane.tar
var saved []byte
// ErrEmpty means this installer carries no control-plane image.
//
// A separate error rather than a message, so the caller can refuse in preflight — before a
// machine has been touched — instead of discovering it at the load, after the runtime has been
// probed and a bundle has been written.
var ErrEmpty = errors.New(
"this mesh-bootstrap carries no control-plane image, so it cannot raise a mesh. A release " +
"build embeds one: `make bootstrap IMAGE=<image>` in the mesh-host repository, where " +
"<image> is a control-plane image already built from the mesh-control source")
// IsEmpty reports whether anything was built in.
//
// It asks whether the bytes are a tar rather than comparing them against the placeholder's text,
// because the question that matters is "can this be loaded", and a truncated or corrupted embed
// answers no to that while matching no placeholder. A tar's first header carries the string
// `ustar` at offset 257 and nothing else does by accident.
func IsEmpty() bool {
const magicAt, magic = 257, "ustar"
return len(saved) < magicAt+len(magic) || string(saved[magicAt:magicAt+len(magic)]) != magic
}
// Saved returns the embedded tar, for loading into a container runtime.
func Saved() ([]byte, error) {
if IsEmpty() {
return nil, ErrEmpty
}
return saved, nil
}
// manifestEntry is the part of a saved image's `manifest.json` this needs.
//
// One field. The layers are the runtime's business and the repository tags are decoration — what
// is wanted is the config, because the digest of the config IS the image id.
type manifestEntry struct {
Config string `json:"Config"`
RepoTags []string `json:"RepoTags"`
}
// ArchiveID is what this archive calls the image it holds. **It is not necessarily the id the
// runtime will assign when the archive is loaded, and it must never be what a bundle names.**
//
// This was called ID, and its comment said it was the id the runtime would give the image once
// loaded. That was wrong, and wrong in the worst available way: right on the machine the image
// was saved on, and wrong on the machine it was carried to. An image id is the sha256 of the
// image's *configuration document*, and a runtime rewrites that document as it loads — a newer
// Docker saves in one format and an older one stores it in another. Same layers, same program,
// different name. Measured on a live raise:
//
// saved on the workstation sha256:b86bb81ca2f9691f24f4725f50962d1e49c98c5ffe211113241243d42d18ceea
// loaded on the machine sha256:2dc219046c73702fc640317f0342a28ec962ef1e9ef547b2f02861c508ca78fb
//
// A bundle naming this id would then name an image the machine does not hold — and nothing serves
// an image named by the digest of its own configuration, which is the whole point of naming one
// that way, so the apply would stop at a pull that cannot succeed. The id a bundle names is read
// back from the runtime after the load (`internal/bootstrap`.Load), which is the only place it is
// a fact rather than a prediction.
//
// What it is still good for is a statement about the FILE — which build somebody embedded — and
// as a hint printed beside the runtime's answer when the two differ, so a person can see that
// they did.
//
// `manifest.json` names the configuration document by its digest — as `<64hex>.json` in the older
// layout and `blobs/sha256/<64hex>` in the OCI one — so both forms reduce to the same sixty-four
// characters.
func ArchiveID(saved []byte) (string, error) {
manifest, err := fileFromTar(saved, "manifest.json")
if err != nil {
return "", err
}
var entries []manifestEntry
if err := json.Unmarshal(manifest, &entries); err != nil {
return "", fmt.Errorf(
"the carried image has a manifest.json this cannot read, so the image it holds "+
"cannot be named: %w", err)
}
// One image, deliberately. A tar holding several would leave the installer choosing which
// one is the control plane, and a bootstrap must not be the thing that guesses.
if len(entries) != 1 {
return "", fmt.Errorf(
"the carried image holds %d images, and the installer raises exactly one control "+
"plane. Save a single image: `docker save --output … <image>`", len(entries))
}
digest := strings.TrimSuffix(path.Base(entries[0].Config), ".json")
if !isSHA256(digest) {
return "", fmt.Errorf(
"the carried image names its configuration %q, which is not a sha256 digest. An "+
"image id is the digest of that configuration, so there is nothing to call this "+
"image", entries[0].Config)
}
return "sha256:" + digest, nil
}
// Tags is what the saved image was called when it was saved.
//
// **Not decoration any more, and this comment used to say it was.** A tag is exactly what a pinned
// bundle may not rely on (novox/hq ADR 0006) and none of this ever reaches a bundle — but the tag
// is how the installer ASKS the runtime what id it assigned, because the id itself is not
// knowable beforehand (see ArchiveID). It is the one name that survives `docker save` and
// `docker load` unchanged, which is precisely what the id does not.
//
// It also still says which build somebody embedded, which is otherwise sixty-four hex characters.
func Tags(saved []byte) []string {
manifest, err := fileFromTar(saved, "manifest.json")
if err != nil {
return nil
}
var entries []manifestEntry
if err := json.Unmarshal(manifest, &entries); err != nil || len(entries) == 0 {
return nil
}
return entries[0].RepoTags
}
func fileFromTar(archive []byte, want string) ([]byte, error) {
reader := tar.NewReader(bytes.NewReader(archive))
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil, fmt.Errorf(
"the carried image has no %s, so it is not something `docker save` produced. "+
"Embed the output of `docker save`, not a layer or a build context", want)
}
if err != nil {
return nil, fmt.Errorf("the carried image cannot be read as a tar: %w", err)
}
if path.Clean(header.Name) == want {
return io.ReadAll(reader)
}
}
}
func isSHA256(s string) bool {
if len(s) != 64 {
return false
}
for _, c := range s {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
internal/image/image_test.go
+160
View File
@@ -0,0 +1,160 @@
package image
import (
"archive/tar"
"bytes"
"encoding/json"
"strings"
"testing"
)
// Each test names the decision it defends (novox/hq ADR 0017).
// savedImage builds what `docker save` produces, as far as this package reads it.
func savedImage(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
for name, content := range files {
header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))}
if err := writer.WriteHeader(header); err != nil {
t.Fatalf("building the fixture: %v", err)
}
if _, err := writer.Write([]byte(content)); err != nil {
t.Fatalf("building the fixture: %v", err)
}
}
if err := writer.Close(); err != nil {
t.Fatalf("building the fixture: %v", err)
}
return buffer.Bytes()
}
func manifest(t *testing.T, config string, tags ...string) string {
t.Helper()
raw, err := json.Marshal([]manifestEntry{{Config: config, RepoTags: tags}})
if err != nil {
t.Fatalf("building the fixture: %v", err)
}
return string(raw)
}
// The archive's own id is read from the FILE, in both layouts `docker save` has used.
//
// **This test used to say that reading it here was what made the load idempotent — that knowing
// the id in advance let the installer ask "do you already hold exactly this". That was wrong.** An
// id is the digest of the image's configuration document, and a runtime rewrites that document as
// it loads, so this is a fact about the archive and not a prediction about any machine. What the
// installer asks a runtime by is the TAG, and what a bundle names is the answer the runtime gives
// back (`internal/bootstrap`.Load).
//
// It is still read and still checked, because it is what says which build somebody embedded — and
// because printing it beside the runtime's answer is how a person sees that the two differ.
func TestTheArchivesOwnIdIsReadFromTheSavedFile(t *testing.T) {
digest := strings.Repeat("a", 64)
// Both layouts `docker save` has used. The older one names the config `<digest>.json`; the OCI
// one names it `blobs/sha256/<digest>`. They carry the same sixty-four characters, and a
// reader that understood only one would work until somebody upgraded their runtime.
for _, config := range []string{digest + ".json", "blobs/sha256/" + digest} {
saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)})
id, err := ArchiveID(saved)
if err != nil {
t.Fatalf("config %q: %v", config, err)
}
if id != "sha256:"+digest {
t.Errorf("config %q gave id %q, want sha256:%s", config, id, digest)
}
}
}
// The tag is read, and it is not decoration: it is the name the installer asks a runtime by,
// because it is the one thing that survives `docker save` and `docker load` unchanged. The id
// does not.
func TestTheSavedTagsAreRead(t *testing.T) {
saved := savedImage(t, map[string]string{
"manifest.json": manifest(t, strings.Repeat("b", 64)+".json", "mesh-control:v1"),
})
got := Tags(saved)
if len(got) != 1 || got[0] != "mesh-control:v1" {
t.Errorf("tags = %v, want [mesh-control:v1]", got)
}
}
// A tar that is not a saved image is refused with what is wrong, not with a nil id.
//
// Nothing here reaches a bundle any more, but this is still the earliest moment somebody can be
// told they embedded the wrong file — and the alternative is finding out at the load, from a
// container runtime, in a sentence about a tar rather than about what was built.
func TestSomethingThatIsNotASavedImageIsRefused(t *testing.T) {
notAnImage := savedImage(t, map[string]string{"hello": "world"})
if _, err := ArchiveID(notAnImage); err == nil {
t.Error("a tar with no manifest.json was accepted as a saved image")
} else if !strings.Contains(err.Error(), "docker save") {
t.Errorf("the refusal does not say what to embed instead: %v", err)
}
if _, err := ArchiveID([]byte("this is not a tar at all")); err == nil {
t.Error("bytes that are not a tar were accepted")
}
}
// Exactly one image. A bootstrap that chose between several would be the thing that guesses which
// one is the control plane, and it would guess right until the day somebody saved two.
func TestATarHoldingSeveralImagesIsRefused(t *testing.T) {
entries, err := json.Marshal([]manifestEntry{
{Config: strings.Repeat("a", 64) + ".json"},
{Config: strings.Repeat("b", 64) + ".json"},
})
if err != nil {
t.Fatal(err)
}
saved := savedImage(t, map[string]string{"manifest.json": string(entries)})
if _, err := ArchiveID(saved); err == nil {
t.Error("a tar holding two images was accepted")
}
}
// An id is a digest or it is nothing. A truncated one names several images, and which one ran
// would be whichever the runtime matched first — the same reasoning `internal/declaration` gives
// for refusing a short image reference.
func TestAConfigThatIsNotADigestIsRefused(t *testing.T) {
for _, config := range []string{"config.json", "abc.json", "blobs/sha256/" + strings.Repeat("a", 63)} {
saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)})
if _, err := ArchiveID(saved); err == nil {
t.Errorf("config %q was accepted and is not a digest", config)
}
}
}
// The committed placeholder must read as "carries nothing", so an installer built from a plain
// checkout says so in preflight rather than getting a machine as far as a running store and
// stopping. This is the same guarantee `internal/bundle` makes about a lock file of only comments.
func TestAnInstallerBuiltFromAPlainCheckoutCarriesNothing(t *testing.T) {
if !IsEmpty() {
// Not a failure of this checkout: `make bootstrap` embeds a real image and puts the
// placeholder back, so a real image here means a build was interrupted.
t.Skip("this checkout has a saved image embedded, so there is no placeholder to check")
}
if _, err := Saved(); err == nil {
t.Fatal("an installer carrying only the placeholder reported it carries an image")
}
}
// And "empty" is decided by whether the bytes could be loaded, not by matching the placeholder's
// text. A truncated or corrupted embed is equally unloadable and equally worth refusing early.
func TestEmptyMeansUnloadableRatherThanEqualToThePlaceholder(t *testing.T) {
restore := saved
defer func() { saved = restore }()
saved = []byte("half a tar, cut off")
if !IsEmpty() {
t.Error("bytes that are not a tar were reported as a carried image")
}
saved = savedImage(t, map[string]string{
"manifest.json": manifest(t, strings.Repeat("c", 64)+".json"),
})
if IsEmpty() {
t.Error("a real saved image was reported as no image at all")
}
}