472 lines
21 KiB
Go
472 lines
21 KiB
Go
// Command mesh-bootstrap brings a mesh into existence on a bare machine.
|
|
//
|
|
// Tier 0, beside `mesh-host` and not inside it. Bootstrapping is done by hand and it changes a
|
|
// machine, which is what tier 0 is (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) — but
|
|
// `mesh-host` states of itself that it connects to nothing and listens on nothing and that what it
|
|
// applies comes from a file, and that is the whole reason an always-running root daemon can be
|
|
// audited by reading one page. An installer that loads images and interrogates a control plane
|
|
// cannot be folded into it without making that sentence false. Same tier, same repository,
|
|
// different program.
|
|
//
|
|
// Genesis is a pivot (novox/hq ADR 0067). It raises a foundation whose control plane is named by the
|
|
// digest of its own configuration — legal exactly where nothing could have served an image — then
|
|
// enrols this machine, installs the registry module, pushes that image into it to get the manifest
|
|
// digest it has never had, reinstalls the control plane as an ordinary module pinned to it, and
|
|
// drops the temporary one. Without --catalog it stops after the foundation and says why.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"strconv"
|
|
"syscall"
|
|
"time"
|
|
|
|
"bufio"
|
|
"github.com/novox/mesh-host/internal/apply"
|
|
"github.com/novox/mesh-host/internal/bootstrap"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
"strings"
|
|
)
|
|
|
|
// version is stamped at build time. Unset in a development build, and said so rather than
|
|
// defaulted to something that looks like a release.
|
|
var version = "development build"
|
|
|
|
const (
|
|
defaultTemplate = "foundation.lock"
|
|
defaultOut = "/var/lib/mesh-host/foundation.lock"
|
|
defaultRegistry = "127.0.0.1:5000"
|
|
defaultHost = "/usr/local/bin/mesh-host"
|
|
// The unit this project actually packages, in `packaging/`. It said `mesh-host.service`, which
|
|
// is not a unit anybody installs — so on a machine with the packaged unit the installer looked
|
|
// for something absent, and told the operator a real machine needs it installed when it was.
|
|
defaultService = "nox-mesh-host.service"
|
|
)
|
|
|
|
const usage = `mesh-bootstrap — make a bare machine into a mesh
|
|
|
|
bootstrap the eighteen steps below (the default)
|
|
version
|
|
|
|
1 preflight what has to be true before anything is changed
|
|
2 load the builder's image, carried in this installer
|
|
3 build the control plane, from its own repository and a commit
|
|
4 bundle the foundation, named for this machine
|
|
5 apply raise it
|
|
6 verify it is up, and the control plane replies
|
|
7 enrol this machine becomes the mesh's first node
|
|
8 registry install the module that gives this mesh an image store
|
|
9 publish push the control plane's image into it, for its first digest
|
|
10 control reinstall the control plane as an ordinary module, pinned to that digest
|
|
11 retire drop the temporary control plane; the host removes it
|
|
12 builder publish the carried builder and install it, so this mesh can
|
|
make the rest of the catalogue rather than be handed it
|
|
|
|
Twelve make a mesh that RUNS. The rest make one that WORKS, asking where a
|
|
human must choose — a run without a terminal answers with the flags below:
|
|
|
|
13 base build the shared toolchain and runtime everything with code
|
|
stands on
|
|
14 store build and install postgres — a database provider, which the
|
|
foundation's own store is not
|
|
15 catalogue build and install the module graph
|
|
16 network choose the private network (--private-network), place this
|
|
machine as its hub (--endpoint, --site)
|
|
17 filter choose the packet filter (--packet-filter) — required, so the
|
|
question is which, not whether
|
|
18 extras anything beyond the floor (--extras)
|
|
|
|
--bundle the foundation template to build this machine's bundle from
|
|
(default ` + defaultTemplate + `)
|
|
--out where the produced bundle is written, for a person to read
|
|
(default ` + defaultOut + `)
|
|
--state where this node records what it has applied
|
|
(default ` + store.DefaultPath + `)
|
|
--source the repository the control plane is built from, on a mesh that
|
|
already exists — not the one being raised
|
|
--source-ref the commit to build. A branch is a moving target somebody else
|
|
controls, and what is cloned here is the trust anchor for
|
|
everything this mesh will ever run
|
|
--source-path the module's directory inside that repository, if not its root
|
|
--catalog a checkout of the mesh's catalogue, holding the registry's, the
|
|
control plane's and the builder's manifests. Without it this stops
|
|
after step 6
|
|
--node the name this machine is known by (default: its hostname)
|
|
--registry where this mesh keeps its own images (default ` + defaultRegistry + `)
|
|
every node pulls the control plane from this, so on a mesh of more
|
|
than one machine it must be an address the others can reach
|
|
--host the mesh-host binary on this machine (default ` + defaultHost + `)
|
|
--host-service the unit that supervises it (default ` + defaultService + `)
|
|
--host-in-background start the host unsupervised instead. It does not survive
|
|
a reboot. This is what a lab does and what no real machine should
|
|
--system which operating system this is; by default it is asked
|
|
--timeout how long any single probe may take (default 30s)
|
|
--wait how long a thing that is merely starting is given (default 3m)
|
|
--dry-run everything that does not change the machine
|
|
--json machine-readable output
|
|
|
|
--tools-source the repository the shared base is built from
|
|
--tools-ref what of it to build (default main)
|
|
--catalog-source the catalogue REPOSITORY, for building its modules;
|
|
--catalog is the checkout that says what they are
|
|
--catalog-ref what of it to build (default main)
|
|
--sdk-source the repository the shared library is built from
|
|
--sdk-ref what of it to build (default main)
|
|
--private-network which private network to run (wireguard)
|
|
--endpoint host:port other machines dial for it; derived from the
|
|
broker address when unsaid
|
|
--site where this machine sits (default main)
|
|
--packet-filter which packet filter to run (nftables)
|
|
--extras catalogue modules beyond the floor, comma-separated
|
|
|
|
The foundation's ports are this machine's, each checked free before anything is
|
|
raised and kept as the node's setting for the module that binds it:
|
|
--store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672
|
|
--registry-port 5000 (follows --registry, and must agree with it)
|
|
--packages-port 3000 --hub-port 51820/udp
|
|
--overlay-range the private network's range (default 10.42.0.0/16); refused
|
|
if it overlaps an interface or route the machine already has
|
|
|
|
--adopted raise a machine in use as an adopted node: what it runs and its
|
|
firewall stay as they are, the foundation's filter is not loaded and
|
|
the mesh guards its own ports instead, and each module is taken on it
|
|
one at a time. Without it, a machine in use is refused
|
|
|
|
The installer carries a builder, not a control plane. What raises a mesh is therefore
|
|
the same thing that will maintain it, and the control plane a mesh ends up running is
|
|
one it built itself, from a repository and a commit it can name and build again.
|
|
|
|
Genesis is a pivot: a temporary control plane installs the registry that makes it
|
|
permanent. The temporary one is called temp-mesh-controller and the permanent one is
|
|
called mesh-controller, so they are two containers with two owners and there is nothing
|
|
to hand over.
|
|
|
|
Every step is idempotent: run it again after fixing whatever it named, and the steps
|
|
that already succeeded say so.
|
|
`
|
|
|
|
func main() {
|
|
// Ctrl-C must stop the installer, not be swallowed by whatever it is waiting for — and it
|
|
// waits on pulls, on a runtime starting, and on a control plane opening its stores.
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
command, opts, jsonOut, err := parseArgs(os.Args[1:])
|
|
if err == nil {
|
|
err = run(ctx, command, opts, jsonOut)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-bootstrap: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
// parseArgs takes an optional subcommand first, then its flags.
|
|
//
|
|
// Parsed in a loop for the reason `mesh-host` records: the standard library stops at the FIRST
|
|
// non-flag argument, so a flag sitting after one is silently dropped and the command exits zero
|
|
// having ignored what it was asked. That fault has been paid for twice in this repository and is
|
|
// not being paid for a third time.
|
|
func parseArgs(args []string) (string, bootstrap.Options, bool, error) {
|
|
opts := bootstrap.Options{
|
|
Template: defaultTemplate,
|
|
Out: defaultOut,
|
|
State: store.DefaultPath,
|
|
Registry: defaultRegistry,
|
|
Host: defaultHost,
|
|
// The machine's own name, because that is what a person already calls it and an installer
|
|
// inventing a different one would leave the mesh naming a machine nobody recognises. It is
|
|
// read here rather than inside the bootstrap so that --node overrides a fact rather than a
|
|
// default computed halfway through.
|
|
Node: hostname(),
|
|
HostService: defaultService,
|
|
// Longer than the host's 10s: these probes reach a container runtime that may be busy
|
|
// pulling, and a probe that times out on a working machine is a false refusal.
|
|
Ports: bootstrap.DefaultPorts(),
|
|
OverlayRange: bootstrap.DefaultOverlayRange,
|
|
Timeout: 30 * time.Second,
|
|
// A socket-activated runtime queued behind the network, and a control plane running its
|
|
// first `initdb`-shaped wait, are both minutes rather than seconds.
|
|
Wait: 3 * time.Minute,
|
|
}
|
|
var jsonOut bool
|
|
|
|
command := "bootstrap"
|
|
if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' {
|
|
command = args[0]
|
|
args = args[1:]
|
|
}
|
|
|
|
set := newFlagSet(&opts, &jsonOut)
|
|
var positionals []string
|
|
rest := args
|
|
for {
|
|
if err := set.Parse(rest); err != nil {
|
|
return "", opts, false, err
|
|
}
|
|
rest = set.Args()
|
|
if len(rest) == 0 {
|
|
break
|
|
}
|
|
positionals = append(positionals, rest[0])
|
|
rest = rest[1:]
|
|
}
|
|
|
|
// Refused rather than ignored: a mistyped argument that changes nothing and reports success is
|
|
// worse than an error, and this program's whole job is to change a machine.
|
|
if len(positionals) > 0 {
|
|
return "", opts, false, fmt.Errorf(
|
|
"unexpected argument %q — try `mesh-bootstrap help`", positionals[0])
|
|
}
|
|
if err := registryAgrees(set, &opts); err != nil {
|
|
return "", opts, false, err
|
|
}
|
|
return command, opts, jsonOut, nil
|
|
}
|
|
|
|
// registryAgrees makes --registry and --registry-port say one port (novox/hq ADR 0100): the
|
|
// registry is raised on the port the node gives it, and every node pulls from the address given.
|
|
// Either may be said alone and the other follows; said both ways, they must agree.
|
|
func registryAgrees(set *flag.FlagSet, opts *bootstrap.Options) error {
|
|
said := map[string]bool{}
|
|
set.Visit(func(f *flag.Flag) { said[f.Name] = true })
|
|
host, portText, err := net.SplitHostPort(opts.Registry)
|
|
if err != nil {
|
|
return fmt.Errorf("--registry %q is not host:port: %w", opts.Registry, err)
|
|
}
|
|
port, err := strconv.Atoi(portText)
|
|
if err != nil {
|
|
return fmt.Errorf("--registry %q does not end in a port", opts.Registry)
|
|
}
|
|
switch {
|
|
case said["registry-port"] && said["registry"] && port != opts.Ports.Registry:
|
|
return fmt.Errorf("--registry %s and --registry-port %d name two ports for one registry",
|
|
opts.Registry, opts.Ports.Registry)
|
|
case said["registry-port"]:
|
|
opts.Registry = net.JoinHostPort(host, strconv.Itoa(opts.Ports.Registry))
|
|
case said["registry"]:
|
|
opts.Ports.Registry = port
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
|
set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError)
|
|
set.SetOutput(os.Stderr)
|
|
set.Usage = func() { fmt.Fprint(os.Stderr, usage) }
|
|
set.StringVar(&opts.Template, "bundle", opts.Template, "the foundation template to build from")
|
|
set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written")
|
|
set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied")
|
|
set.StringVar(&opts.Catalogue, "catalog", opts.Catalogue,
|
|
"a checkout of the mesh's catalogue, for the registry's and the builder's manifests and what phase two installs; without it this stops after the foundation")
|
|
set.StringVar(&opts.Source.Repository, "source", opts.Source.Repository,
|
|
"the repository the control plane is built from, on a mesh that already exists")
|
|
set.StringVar(&opts.Source.Ref, "source-ref", opts.Source.Ref,
|
|
"the commit to build; a branch is a moving target somebody else controls")
|
|
set.StringVar(&opts.Source.Path, "source-path", opts.Source.Path,
|
|
"the module's directory inside that repository, if not its root")
|
|
set.StringVar(&opts.Node, "node", opts.Node, "the name this machine is known by")
|
|
set.StringVar(&opts.Registry, "registry", opts.Registry, "where this mesh keeps its own images")
|
|
set.StringVar(&opts.Host, "host", opts.Host, "the mesh-host binary on this machine")
|
|
set.StringVar(&opts.HostService, "host-service", opts.HostService, "the unit that supervises it")
|
|
set.BoolVar(&opts.HostInBackground, "host-in-background", false,
|
|
"start the host unsupervised; it does not survive a reboot")
|
|
set.StringVar(&opts.System, "system", opts.System, "which operating system this is")
|
|
set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take")
|
|
set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given")
|
|
set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine")
|
|
set.BoolVar(jsonOut, "json", false, "machine-readable output")
|
|
|
|
// Phase two — the installer goes as far as it can, and asks where a human must choose. A run
|
|
// without a terminal answers with these; a required choice nothing answered is a refusal.
|
|
set.StringVar(&opts.ToolsSource.Repository, "tools-source", opts.ToolsSource.Repository,
|
|
"the repository the shared base is built from")
|
|
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
|
|
"what of it to build (default main)")
|
|
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
|
|
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
|
|
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
|
|
"what of it to build (default main)")
|
|
set.StringVar(&opts.SDKSource.Repository, "sdk-source", opts.SDKSource.Repository,
|
|
"the repository the shared library is built from, published before the base resolves it")
|
|
set.StringVar(&opts.SDKSource.Ref, "sdk-ref", opts.SDKSource.Ref,
|
|
"what of it to build (default main)")
|
|
set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network")
|
|
|
|
// The foundation's ports are this node's (novox/hq ADR 0100): each is checked free before
|
|
// anything is raised, and becomes the node's setting for the module that binds it.
|
|
for _, p := range []struct {
|
|
name, what string
|
|
into *int
|
|
}{
|
|
{"store-port", "the store", &opts.Ports.Store},
|
|
{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
|
|
{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
|
|
{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
|
|
{"registry-port", "the registry", &opts.Ports.Registry},
|
|
{"packages-port", "the package registry", &opts.Ports.Packages},
|
|
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
|
|
} {
|
|
set.IntVar(p.into, p.name, *p.into, "the machine's port for "+p.what)
|
|
}
|
|
set.BoolVar(&opts.Adopted, "adopted", false,
|
|
"raise this machine adopted: keep what it runs and its firewall until each module is taken")
|
|
set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange,
|
|
"the private network's address range; must not overlap a tunnel the machine already runs")
|
|
if opts.Answers == nil {
|
|
opts.Answers = map[string]string{}
|
|
}
|
|
answers := opts.Answers
|
|
set.Func("private-network", "which private network to run (wireguard)", func(v string) error {
|
|
answers["private-network"] = v
|
|
return nil
|
|
})
|
|
set.Func("packet-filter", "which packet filter to run (nftables)", func(v string) error {
|
|
answers["packet-filter"] = v
|
|
return nil
|
|
})
|
|
set.Func("endpoint", "host:port other machines dial for the private network (derived from the broker address if unsaid)", func(v string) error {
|
|
answers["endpoint"] = v
|
|
return nil
|
|
})
|
|
set.Func("extras", "catalogue modules beyond the floor, comma-separated", func(v string) error {
|
|
for _, e := range strings.Split(v, ",") {
|
|
if e = strings.TrimSpace(e); e != "" {
|
|
opts.Extras = append(opts.Extras, e)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return set
|
|
}
|
|
|
|
// askOn is how a person is asked a choice, when there is a person: the question, the options, a
|
|
// read line. Wired only when stdin is a terminal, so the lab and unattended runs are never left
|
|
// waiting on a prompt nobody will answer.
|
|
func askOn(in *bufio.Reader, out io.Writer) func(bootstrap.Choice) (string, error) {
|
|
return func(c bootstrap.Choice) (string, error) {
|
|
fmt.Fprintf(out, "\n%s\n", c.Question)
|
|
if len(c.Options) > 0 {
|
|
fmt.Fprintf(out, " options: %s\n", strings.Join(c.Options, ", "))
|
|
}
|
|
if c.Default != "" {
|
|
fmt.Fprintf(out, " [%s] ", c.Default)
|
|
} else {
|
|
fmt.Fprint(out, " > ")
|
|
}
|
|
line, err := in.ReadString('\n')
|
|
if err != nil {
|
|
return "", fmt.Errorf("the terminal went away mid-question: %w", err)
|
|
}
|
|
return strings.TrimSpace(line), nil
|
|
}
|
|
}
|
|
|
|
func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error {
|
|
switch command {
|
|
case "bootstrap":
|
|
say := func(line string) {
|
|
if !jsonOut {
|
|
fmt.Println(line)
|
|
}
|
|
}
|
|
// A person at a terminal is asked the choices; anything else answers with flags. `--json`
|
|
// counts as "anything else": a run whose output is being parsed has no one reading a
|
|
// question.
|
|
if info, err := os.Stdin.Stat(); err == nil &&
|
|
info.Mode()&os.ModeCharDevice != 0 && !jsonOut {
|
|
opts.Prompt = askOn(bufio.NewReader(os.Stdin), os.Stdout)
|
|
}
|
|
result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{
|
|
Run: apply.ExecRunner,
|
|
Dial: dial,
|
|
Fetch: fetch,
|
|
}, say)
|
|
|
|
// Printed whichever way it went. What the installer got through before it stopped is on
|
|
// the machine either way, and a report that only exists on success describes a machine
|
|
// nobody has (novox/hq ADR 0018).
|
|
if jsonOut {
|
|
encoder := json.NewEncoder(os.Stdout)
|
|
encoder.SetIndent("", " ")
|
|
if encodeErr := encoder.Encode(result); encodeErr != nil && err == nil {
|
|
return encodeErr
|
|
}
|
|
}
|
|
return err
|
|
|
|
case "version":
|
|
fmt.Println(version)
|
|
return nil
|
|
|
|
case "help", "-h", "--help":
|
|
fmt.Fprint(os.Stderr, usage)
|
|
return nil
|
|
|
|
default:
|
|
return fmt.Errorf("unknown command %q — try `mesh-bootstrap help`", command)
|
|
}
|
|
}
|
|
|
|
// hostname is what this machine calls itself, or empty.
|
|
//
|
|
// Empty rather than a guess: a machine that cannot say its own name is one the installer must be
|
|
// told about, and `mesh-bootstrap-0` would be a name in the mesh's records that matches nothing
|
|
// anybody types anywhere else. The refusal happens at step 6, where the name is first needed.
|
|
func hostname() string {
|
|
name, err := os.Hostname()
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return name
|
|
}
|
|
|
|
// fetch asks an HTTP endpoint and reports what it said.
|
|
//
|
|
// Plain HTTP, and only at the mesh's own registry: it is reached over the mesh's private network,
|
|
// which is already the encrypted and authenticated thing, and a second layer inside it would be
|
|
// certificates to issue and rotate for no property the first does not have (mesh-controller's
|
|
// `internal/builder` pushes to it on the same reasoning).
|
|
//
|
|
// The body is read with a limit. What is asked for is a status and a short JSON answer, and a
|
|
// registry that answered with a gigabyte would otherwise be an installer that never returns.
|
|
func fetch(ctx context.Context, url string) (int, string, error) {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
response, err := http.DefaultClient.Do(request)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
defer response.Body.Close()
|
|
|
|
said, err := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
|
if err != nil {
|
|
return response.StatusCode, "", err
|
|
}
|
|
return response.StatusCode, string(said), nil
|
|
}
|
|
|
|
// dial answers whether a TCP address responds.
|
|
//
|
|
// A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a
|
|
// connection to exactly this address. A machine whose DNS resolves and whose route is missing
|
|
// passes a lookup and fails the thing that matters.
|
|
func dial(ctx context.Context, address string) error {
|
|
var dialer net.Dialer
|
|
conn, err := dialer.DialContext(ctx, "tcp", address)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return conn.Close()
|
|
}
|