Self-upgrade installer: genesis pivot, rename, adopt store+broker #13

Merged
jschoubben merged 12 commits from feat/a-bed-that-hands-over-nothing into main 2026-09-16 21:22:07 +00:00
7 changed files with 550 additions and 4 deletions
Showing only changes of commit 21474b0144 - Show all commits
+94 -1
View File
@@ -28,9 +28,11 @@ import (
"syscall"
"time"
"bufio"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/bootstrap"
"github.com/novox/mesh-host/internal/store"
"strings"
)
// version is stamped at build time. Unset in a development build, and said so rather than
@@ -50,7 +52,7 @@ const (
const usage = `mesh-bootstrap — make a bare machine into a mesh
bootstrap the twelve steps below (the default)
bootstrap the eighteen steps below (the default)
version
1 preflight what has to be true before anything is changed
@@ -67,6 +69,20 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
12 builder publish the carried builder and install it, so this mesh can
make the rest of the catalogue rather than be handed it
Twelve make a mesh that RUNS. The rest make one that WORKS, asking where a
human must choose — a run without a terminal answers with the flags below:
13 base build the shared toolchain and runtime everything with code
stands on
14 store build and install postgres — a database provider, which the
substrate's own store is not
15 catalogue build and install the module graph
16 network choose the private network (--private-network), place this
machine as its hub (--endpoint, --site)
17 filter choose the packet filter (--packet-filter) — required, so the
question is which, not whether
18 extras anything beyond the floor (--extras)
--bundle the substrate template to build this machine's bundle from
(default ` + defaultTemplate + `)
--out where the produced bundle is written, for a person to read
@@ -96,6 +112,18 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
--dry-run everything that does not change the machine
--json machine-readable output
--tools-source the repository the shared base is built from
--tools-ref what of it to build (default main)
--catalog-source the catalogue REPOSITORY, for building its modules;
--catalog is the checkout that says what they are
--catalog-ref what of it to build (default main)
--private-network which private network to run (wireguard)
--endpoint host:port other machines dial for it; derived from the
broker address when unsaid
--site where this machine sits (default main)
--packet-filter which packet filter to run (nftables)
--extras catalogue modules beyond the floor, comma-separated
The installer carries a builder, not a control plane. What raises a mesh is therefore
the same thing that will maintain it, and the control plane a mesh ends up running is
one it built itself, from a repository and a commit it can name and build again.
@@ -209,9 +237,67 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given")
set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine")
set.BoolVar(jsonOut, "json", false, "machine-readable output")
// Phase two — the installer goes as far as it can, and asks where a human must choose. A run
// without a terminal answers with these; a required choice nothing answered is a refusal.
set.StringVar(&opts.ToolsSource.Repository, "tools-source", opts.ToolsSource.Repository,
"the repository the shared base is built from")
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
"what of it to build (default main)")
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
"what of it to build (default main)")
set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network")
if opts.Answers == nil {
opts.Answers = map[string]string{}
}
answers := opts.Answers
set.Func("private-network", "which private network to run (wireguard)", func(v string) error {
answers["private-network"] = v
return nil
})
set.Func("packet-filter", "which packet filter to run (nftables)", func(v string) error {
answers["packet-filter"] = v
return nil
})
set.Func("endpoint", "host:port other machines dial for the private network (derived from the broker address if unsaid)", func(v string) error {
answers["endpoint"] = v
return nil
})
set.Func("extras", "catalogue modules beyond the floor, comma-separated", func(v string) error {
for _, e := range strings.Split(v, ",") {
if e = strings.TrimSpace(e); e != "" {
opts.Extras = append(opts.Extras, e)
}
}
return nil
})
return set
}
// askOn is how a person is asked a choice, when there is a person: the question, the options, a
// read line. Wired only when stdin is a terminal, so the lab and unattended runs are never left
// waiting on a prompt nobody will answer.
func askOn(in *bufio.Reader, out io.Writer) func(bootstrap.Choice) (string, error) {
return func(c bootstrap.Choice) (string, error) {
fmt.Fprintf(out, "\n%s\n", c.Question)
if len(c.Options) > 0 {
fmt.Fprintf(out, " options: %s\n", strings.Join(c.Options, ", "))
}
if c.Default != "" {
fmt.Fprintf(out, " [%s] ", c.Default)
} else {
fmt.Fprint(out, " > ")
}
line, err := in.ReadString('\n')
if err != nil {
return "", fmt.Errorf("the terminal went away mid-question: %w", err)
}
return strings.TrimSpace(line), nil
}
}
func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error {
switch command {
case "bootstrap":
@@ -220,6 +306,13 @@ func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bo
fmt.Println(line)
}
}
// A person at a terminal is asked the choices; anything else answers with flags. `--json`
// counts as "anything else": a run whose output is being parsed has no one reading a
// question.
if info, err := os.Stdin.Stat(); err == nil &&
info.Mode()&os.ModeCharDevice != 0 && !jsonOut {
opts.Prompt = askOn(bufio.NewReader(os.Stdin), os.Stdout)
}
result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{
Run: apply.ExecRunner,
Dial: dial,
+2
View File
@@ -108,6 +108,8 @@ func TestTheUsageTextAndTheFlagsAgree(t *testing.T) {
for _, promised := range []string{
"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json",
"catalog", "node", "registry", "host", "host-service", "host-in-background",
"tools-source", "tools-ref", "catalog-source", "catalog-ref",
"private-network", "endpoint", "site", "packet-filter", "extras",
} {
if !declared[promised] {
t.Errorf("the usage text promises --%s and no such flag exists", promised)
+68 -3
View File
@@ -53,6 +53,12 @@ const (
StepControlPlane Step = "control-plane"
StepRetire Step = "retire"
StepBuilder Step = "builder"
StepBase Step = "base"
StepStore Step = "store"
StepCatalogue Step = "catalogue"
StepNetwork Step = "network"
StepFilter Step = "filter"
StepExtras Step = "extras"
)
// Steps in the order they happen, so a failure can say "step 2 of 11".
@@ -69,6 +75,10 @@ const (
var Steps = []Step{
StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify,
StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder,
// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
// build, to say what it holds, on its network, filtering. They used to be things somebody
// typed afterwards, which is how they went missing without anything complaining.
StepBase, StepStore, StepCatalogue, StepNetwork, StepFilter, StepExtras,
}
// Error is a failure, named by the step it happened in.
@@ -145,6 +155,24 @@ type Options struct {
// HostInBackground starts the host unsupervised instead, which is what the lab does and what no
// real machine should do — it does not survive a reboot.
HostInBackground bool
// ---- phase two — a mesh that runs becomes a mesh that works ----
// ToolsSource is where the shared base is built from. Everything with code of its own
// compiles against it, so it is the first thing the mesh builds for itself.
ToolsSource Source
// CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue
// CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it.
CatalogSource Source
// Site is where this machine sits, for the private network's placement.
Site string
// Answers are the choices, answered by flag: name → answer. What a terminal would be asked.
Answers map[string]string
// Prompt asks a person one choice. Nil is an unattended run: flags and defaults answer, and a
// required choice nothing answered is a refusal rather than a guess.
Prompt func(Choice) (string, error)
// Extras are catalogue modules beyond the floor, asked for by name.
Extras []string
}
// pivots reports whether this run goes past the substrate.
@@ -535,9 +563,46 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
return result, failed(StepBuilder, err)
}
say("\nthis machine is a mesh of one node, and the control plane it runs is a module " +
"pinned to an image its own registry serves.")
say("it holds a builder, so it can make the rest of the catalogue rather than be handed it.")
// ---- 13. base -------------------------------------------------------------------------
say("base — the shared toolchain and runtime everything with code stands on")
if err := BuildBase(ctx, o, permanentControl, say); err != nil {
return result, failed(StepBase, err)
}
// ---- 14. store ------------------------------------------------------------------------
// A database PROVIDER. The substrate's store is the control plane's own memory and offers
// nothing to anything; the first thing that wants a database is the catalogue, next.
say("store — a database provider, which the substrate's own store is not")
if err := InstallFromCatalogue(ctx, o, permanentControl, "postgres", say); err != nil {
return result, failed(StepStore, err)
}
// ---- 15. catalogue --------------------------------------------------------------------
say("catalogue — the module graph: what is held, what a change reaches, what to rebuild")
if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil {
return result, failed(StepCatalogue, err)
}
// ---- 16. network ----------------------------------------------------------------------
say("network — the private network, and this machine's name on it")
if err := PlaceOnTheNetwork(ctx, o, permanentControl, rewritten.BrokerAddress, say); err != nil {
return result, failed(StepNetwork, err)
}
// ---- 17. filter -----------------------------------------------------------------------
say("filter — required, so the question is which, not whether")
if err := ChooseAndInstallFilter(ctx, o, permanentControl, say); err != nil {
return result, failed(StepFilter, err)
}
// ---- 18. extras -----------------------------------------------------------------------
say("extras — beyond the floor, if asked")
if err := InstallExtras(ctx, o, permanentControl, say); err != nil {
return result, failed(StepExtras, err)
}
say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " +
"sits on its private network, and filters what modules declared.")
say("what remains is somebody else's: adding nodes, and assigning what they should run.")
return result, nil
+79
View File
@@ -0,0 +1,79 @@
package bootstrap
import (
"fmt"
"strings"
)
// What the installer asks a person, and how an unattended run answers.
//
// **The installer goes as far as it can, and where a human must choose, it asks** — a packet
// filter is required, so the question is not whether but which. A run with a terminal is asked;
// a run without one (the lab, an unattended machine) answers with flags, and a required choice
// with no flag, no terminal and no lone option is a refusal rather than a guess.
// Choice is one question the installer needs answered.
type Choice struct {
// Name is the flag that answers it unattended: --packet-filter, --private-network.
Name string
// Question is what a person is asked, ending with what the options are.
Question string
// Options are the acceptable answers. Empty means free-form (an address, a list).
Options []string
// Default is used when nothing and nobody answered. Empty means the choice is required.
Default string
}
// decide resolves one choice, in the order a person would expect:
//
// an explicit answer (the flag) wins; a lone option answers itself, said aloud; a terminal is
// asked; a default fills in; and a required choice nothing answered is refused naming its flag.
func decide(c Choice, answered string, prompt func(Choice) (string, error), say func(string)) (string, error) {
if got := strings.TrimSpace(answered); got != "" {
return validated(c, got)
}
if len(c.Options) == 1 {
// The only answer there is. Said rather than silent, because "it chose for me" and "there
// was nothing to choose" read identically afterwards unless one of them says so.
say(fmt.Sprintf(" %-17s %s — the only option there is", c.Name, c.Options[0]))
return c.Options[0], nil
}
if prompt != nil {
got, err := prompt(c)
if err != nil {
return "", err
}
if strings.TrimSpace(got) == "" && c.Default != "" {
say(fmt.Sprintf(" %-17s %s (default)", c.Name, c.Default))
return c.Default, nil
}
return validated(c, strings.TrimSpace(got))
}
if c.Default != "" {
say(fmt.Sprintf(" %-17s %s (default)", c.Name, c.Default))
return c.Default, nil
}
return "", fmt.Errorf(
"%s must be chosen and nothing chose it: no --%s, no terminal to ask on%s",
c.Name, c.Name, orOptions(c))
}
func validated(c Choice, got string) (string, error) {
if len(c.Options) == 0 {
return got, nil
}
for _, option := range c.Options {
if got == option {
return got, nil
}
}
return "", fmt.Errorf("%q is not a %s this mesh offers. It has %s",
got, c.Name, strings.Join(c.Options, ", "))
}
func orOptions(c Choice) string {
if len(c.Options) == 0 {
return ""
}
return ". It offers " + strings.Join(c.Options, ", ")
}
+64
View File
@@ -0,0 +1,64 @@
package bootstrap
import (
"strings"
"testing"
)
func quietly(string) {}
// A flag answers, and answers wrongly is refused naming what would have worked.
func TestAFlagAnswersAndAWrongOneIsRefused(t *testing.T) {
filter := Choice{Name: "packet-filter", Options: []string{"nftables", "ufw"}}
got, err := decide(filter, "ufw", nil, quietly)
if err != nil || got != "ufw" {
t.Fatalf("an explicit answer was not taken: %q, %v", got, err)
}
_, err = decide(filter, "iptables", nil, quietly)
if err == nil {
t.Fatal("an answer nothing offers was accepted")
}
if !strings.Contains(err.Error(), "nftables") || !strings.Contains(err.Error(), "ufw") {
t.Fatalf("the refusal does not say what would have worked: %v", err)
}
}
// A lone option answers itself — and says so, because "it chose for me" and "there was nothing to
// choose" read identically afterwards unless one of them speaks.
func TestALoneOptionAnswersItselfAloud(t *testing.T) {
var said []string
got, err := decide(Choice{Name: "private-network", Options: []string{"wireguard"}},
"", nil, func(line string) { said = append(said, line) })
if err != nil || got != "wireguard" {
t.Fatalf("the only option was not taken: %q, %v", got, err)
}
if len(said) == 0 || !strings.Contains(said[0], "only option") {
t.Fatalf("choosing silently: %v", said)
}
}
// A terminal is asked; an empty answer takes the default when there is one.
func TestATerminalIsAskedAndEmptyTakesTheDefault(t *testing.T) {
asked := 0
prompt := func(c Choice) (string, error) { asked++; return "", nil }
got, err := decide(Choice{Name: "extras", Default: "none"}, "", prompt, quietly)
if err != nil || got != "none" || asked != 1 {
t.Fatalf("empty answer at a prompt did not take the default: %q asked=%d %v", got, asked, err)
}
}
// **Unattended and required is a refusal, not a guess.** The lab and any machine without a
// terminal must be answerable by flags — and a required choice with no flag has to stop the run
// naming the flag, because a guessed packet filter is a machine somebody else configured.
func TestUnattendedAndRequiredRefusesNamingTheFlag(t *testing.T) {
_, err := decide(Choice{Name: "packet-filter", Options: []string{"nftables", "ufw"}},
"", nil, quietly)
if err == nil {
t.Fatal("a required choice was guessed for an unattended run")
}
if !strings.Contains(err.Error(), "--packet-filter") {
t.Fatalf("the refusal does not name the flag that answers it: %v", err)
}
}
+218
View File
@@ -0,0 +1,218 @@
package bootstrap
import (
"context"
"encoding/json"
"fmt"
"net"
"strings"
"time"
)
// Phase two — a mesh that runs becomes a mesh that works.
//
// Genesis ends with a control plane, a store, a broker, a registry and a builder — a mesh that
// RUNS. It holds no module graph, has no private network, and filters nothing. Those used to be
// things somebody typed afterwards, which is how they went missing for weeks without anything
// complaining (novox/hq 03-DESIGN/01-to-be/21-the-installation-in-full.md). The installer goes as
// far as it can instead, and asks where a human must choose.
//
// Everything here is `module add`, `build`, `assign` and `push` — the same verbs a person types,
// through the same commands, so what the installer does and what an operator does remain one act.
// buildWait bounds one module build. Generous, because the first build compiles a toolchain.
const buildWait = 20 * time.Minute
// BuildBase asks the mesh to build the shared base every module with code of its own stands on.
//
// **First, because until it exists nothing else with code can be built.** Not registered as a
// module here: it is never assigned — it runs nowhere — and the build itself records what was
// made, which is all anything downstream reads.
func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error {
if o.ToolsSource.Repository == "" {
return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " +
"own repository, and an installer told nothing cannot know where that is")
}
say(" building " + o.ToolsSource.Repository + " at " + refOr(o.ToolsSource.Ref))
_, err := control.within(buildWait).tell(ctx,
"build", o.ToolsSource.Repository, "--ref", refOr(o.ToolsSource.Ref), "--wait", "1200s")
return err
}
// InstallFromCatalogue builds a catalogue module and installs it on this machine.
//
// The order matters and is the one the lab proved: register the manifest, build (so the artifact
// exists before anything resolves it), issue its broker account (a runtime without one starts,
// parses a password as a credential document, and loops), assign, push.
func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane,
module string, say func(string)) error {
manifest, err := readManifest(o.Catalogue, module)
if err != nil {
return err
}
remote := "/" + module + "-module.json"
if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
return err
}
if _, err := control.tell(ctx, "module", "add", remote); err != nil {
return err
}
say(" registered " + module)
if builds(manifest) {
if o.CatalogSource.Repository == "" {
return fmt.Errorf("%s has to be built and there is no --catalog-source to build it "+
"from: the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where "+
"a builder clones it", module)
}
say(" building " + module)
if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository,
"--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref),
"--wait", "1200s"); err != nil {
return err
}
}
if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil {
// Not every module consumes the broker; one that does not is refused an account and that
// is fine. Said rather than silent, so a module that SHOULD have one and was refused is
// visible here rather than as a crash-loop later.
say(" no account " + module + " — it declares nothing to say on the broker")
} else {
say(" account issued " + module)
}
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
return err
}
if _, err := pushNode(ctx, o, control, say); err != nil {
return err
}
say(" installed " + module)
return nil
}
// PlaceOnTheNetwork chooses a private-network provider, assigns it, and places this machine as
// the hub.
//
// **Assigning is not being on the network** — a lesson paid for: the module installed, the names
// file was written with no names in it, and everything reported success, because nobody had said
// where this machine IS. So placement is part of the step, not a separate act.
func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
brokerAddress string, say func(string)) error {
network, err := decide(Choice{
Name: "private-network",
Question: "Which private network should this mesh run?",
Options: []string{"wireguard"},
}, o.Answers["private-network"], o.Prompt, say)
if err != nil {
return err
}
// Today the one provider is the control plane's own computed module. The choice exists so
// that the day there are two, this asks instead of assuming.
module := "networking"
_ = network
endpoint, err := decide(Choice{
Name: "endpoint",
Question: "Where do other machines reach this one for the private network? " +
"(host:port; the host other machines dial)",
Default: derivedEndpoint(brokerAddress),
}, o.Answers["endpoint"], o.Prompt, say)
if err != nil {
return err
}
if endpoint == "" {
return fmt.Errorf("the private network needs an endpoint other machines can dial, and " +
"nothing said one: pass --endpoint, or --broker-address so one can be derived")
}
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
return err
}
if _, err := control.tell(ctx, "overlay", "place", o.Node,
"--hub", "--endpoint", endpoint, "--site", o.Site); err != nil {
return err
}
if _, err := pushNode(ctx, o, control, say); err != nil {
return err
}
say(" on the network " + o.Node + " is the hub, at " + endpoint)
return nil
}
// ChooseAndInstallFilter picks the packet filter — required, so the question is which, not
// whether — and installs it.
func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) error {
filter, err := decide(Choice{
Name: "packet-filter",
Question: "Which packet filter should this machine run?",
Options: []string{"nftables"},
}, o.Answers["packet-filter"], o.Prompt, say)
if err != nil {
return err
}
return InstallFromCatalogue(ctx, o, control, filter, say)
}
// InstallExtras installs what was asked for beyond the floor.
//
// One refusal per act: an extra that cannot be installed fails the run, because somebody asked
// for it by name and a mesh that reports success minus one thing is reporting the wrong thing.
func InstallExtras(ctx context.Context, o Options, control controlPlane, say func(string)) error {
asked, err := decide(Choice{
Name: "extras",
Question: "Anything beyond the floor? (comma-separated catalogue modules — " +
"gitea, step-ca, dnsmasq — or nothing)",
Default: "none",
}, strings.Join(o.Extras, ","), o.Prompt, say)
if err != nil {
return err
}
if asked == "" || asked == "none" {
say(" extras none")
return nil
}
for _, extra := range strings.Split(asked, ",") {
if extra = strings.TrimSpace(extra); extra == "" {
continue
}
if err := InstallFromCatalogue(ctx, o, control, extra, say); err != nil {
return fmt.Errorf("%s was asked for and could not be installed: %w", extra, err)
}
}
return nil
}
// builds says whether a manifest declares anything to build.
func builds(manifest []byte) bool {
var m struct {
Build *struct {
Artifacts []json.RawMessage `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return false
}
return m.Build != nil && len(m.Build.Artifacts) > 0
}
// derivedEndpoint is the default place other machines dial for the private network: the same host
// they already dial for the broker, on WireGuard's ordinary port. One fact, not two.
func derivedEndpoint(brokerAddress string) string {
host, _, err := net.SplitHostPort(brokerAddress)
if err != nil || host == "" {
return ""
}
return net.JoinHostPort(host, "51820")
}
func refOr(ref string) string {
if ref == "" {
return "main"
}
return ref
}
+25
View File
@@ -0,0 +1,25 @@
package bootstrap
import "testing"
// The endpoint other machines dial defaults to the host they already dial — the broker's — on
// WireGuard's port. One fact, not two that drift.
func TestTheEndpointDerivesFromTheBrokerAddress(t *testing.T) {
if got := derivedEndpoint("192.0.2.10:5671"); got != "192.0.2.10:51820" {
t.Fatalf("derived %q", got)
}
if got := derivedEndpoint(""); got != "" {
t.Fatalf("an endpoint was invented from nothing: %q", got)
}
}
// A manifest with artifacts builds; one without does not — which is what separates postgres (a
// bundle to compile) from nftables (a package and a service).
func TestOnlyAManifestWithArtifactsBuilds(t *testing.T) {
if !builds([]byte(`{"build":{"artifacts":[{"name":"x"}]}}`)) {
t.Fatal("a manifest with artifacts was not built")
}
if builds([]byte(`{"resources":[{"id":"p","type":"package","package":"nftables"}]}`)) {
t.Fatal("a manifest with nothing to build was built anyway")
}
}