package bootstrap import ( "context" "encoding/json" "errors" "fmt" "net" "strconv" "strings" "time" ) // Phase two — a mesh that runs becomes a mesh that works. // // Genesis ends with a control plane, a store, a broker, a registry and a builder — a mesh that // RUNS. It holds no module graph, has no private network, and filters nothing. Those used to be // things somebody typed afterwards, which is how they went missing for weeks without anything // complaining (novox/hq 03-DESIGN/01-to-be/21-the-installation-in-full.md). The installer goes as // far as it can instead, and asks where a human must choose. // // Everything here is `module add`, `build`, `assign` and `push` — the same verbs a person types, // through the same commands, so what the installer does and what an operator does remain one act. // buildWait bounds one module build. Generous, because the first build compiles a toolchain. const buildWait = 20 * time.Minute // BuildBase asks the mesh to build the shared base every module with code of its own stands on, and // the toolchain of every language genesis's own modules are written in. // // **First, because until it exists nothing else with code can be built.** Not registered as a // module here: it is never assigned — it runs nowhere — and the build itself records what was // made, which is all anything downstream reads. // // **Every toolchain a genesis module compiles in** (novox/hq issue 616). A bundle is compiled in its // language's base (mesh-controller internal/builder toolchain.go): TypeScript in mesh-tools, Go in // mesh-tools-go. The store's and the bus's modules carry Go code, so a genesis that built only // mesh-tools stopped at the store step: "this mesh holds no copy of it. Build mesh-tools-go first". // The bases are the ones genesisToolchains names, and a test holds them to the languages of every // module genesis installs, read from the catalogue. func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error { if o.ToolsSource.Repository == "" { return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " + "own repository, and an installer told nothing cannot know where that is") } goSource, err := toolsGoSource(o) if err != nil { return err } for _, src := range []Source{o.ToolsSource, goSource} { say(" building " + src.Repository + " at " + refOr(src.Ref)) if _, err := control.within(buildWait).tell(ctx, "build", src.Repository, "--ref", refOr(src.Ref), "--wait", "1200s"); err != nil { return err } } return nil } // genesisToolchains is the base each language's bundles compile in, for the languages genesis's own // modules are written in — the same pairs as mesh-controller internal/builder toolchain.go — in the // order BuildBase builds them. var genesisToolchains = []struct{ Language, Base string }{ {"typescript", "mesh-tools"}, {"go", "mesh-tools-go"}, } // toolsGoSource is where the Go toolchain is built from: as given, or the repository beside the base's // named mesh-tools-go, which is how the mesh keeps its repositories side by side. func toolsGoSource(o Options) (Source, error) { if o.ToolsGoSource.Repository != "" { return o.ToolsGoSource, nil } repo := strings.TrimSuffix(o.ToolsSource.Repository, "/") suffix := "" if strings.HasSuffix(repo, ".git") { repo, suffix = strings.TrimSuffix(repo, ".git"), ".git" } if !strings.HasSuffix(repo, "mesh-tools") { return Source{}, fmt.Errorf("the Go toolchain is built from mesh-tools-go, and --tools-source (%s) is not a "+ "repository named mesh-tools to find it beside; give --tools-go-source", o.ToolsSource.Repository) } return Source{Repository: repo + "-go" + suffix, Ref: o.ToolsGoSource.Ref}, nil } // InstallFromCatalogue builds a catalogue module and installs it on this machine. // // The order matters and is the one the lab proved: register the manifest, build (so the artifact // exists before anything resolves it), issue its broker account (a runtime without one starts, // parses a password as a credential document, and loops), assign, push. func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane, module string, say func(string)) error { manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { return err } if _, err := control.tell(ctx, "module", "add", remote); err != nil { return err } say(" registered " + module) if builds(manifest) { if o.CatalogSource.Repository == "" { return fmt.Errorf("%s has to be built and there is no --catalog-source to build it "+ "from: the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where "+ "a builder clones it", module) } say(" building " + module) if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { return err } } if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { // Not every module consumes the broker; one that does not is refused an account and that // is fine. Said rather than silent, so a module that SHOULD have one and was refused is // visible here rather than as a crash-loop later. say(" no account " + module + " — it declares nothing to say on the broker") } else { say(" account issued " + module) } if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } if err := prepareModule(ctx, o, control, module, say); err != nil { return err } if _, err := pushNode(ctx, o, control, say); err != nil { return err } say(" installed " + module) return nil } // PlaceOnTheNetwork chooses a private-network provider, assigns it, and places this machine as // the hub. // // **Assigning is not being on the network** — a lesson paid for: the module installed, the names // file was written with no names in it, and everything reported success, because nobody had said // where this machine IS. So placement is part of the step, not a separate act. func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, brokerAddress string, say func(string)) error { network, err := decide(Choice{ Name: "private-network", Question: "Which private network should this mesh run?", Options: []string{"wireguard"}, }, o.Answers["private-network"], o.Prompt, say) if err != nil { return err } // Today the one provider is the control plane's own computed module, assigned by its own name: // the `networking` bundle that only required it is retired (novox/hq ADR 0226). The choice // exists so that the day there are two, this asks instead of assuming. module := "mesh-wireguard" _ = network endpoint, err := decide(Choice{ Name: "endpoint", Question: "Where do other machines reach this one for the private network? " + "(host:port; the host other machines dial)", Default: derivedEndpoint(brokerAddress, o.Ports.orDefaults().Hub), }, o.Answers["endpoint"], o.Prompt, say) if err != nil { return err } if endpoint == "" { return fmt.Errorf("the private network needs an endpoint other machines can dial, and " + "nothing said one: pass --endpoint, or --broker-address so one can be derived") } endpoint, err = endpointAgrees(endpoint, o.Ports.orDefaults().Hub) if err != nil { return err } if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } if _, err := control.tell(ctx, "overlay", "place", o.Node, "--hub", "--endpoint", endpoint, "--site", o.Site); err != nil { return err } if _, err := pushNode(ctx, o, control, say); err != nil { return err } say(" on the network " + o.Node + " is the hub, at " + endpoint) return nil } // ChooseAndInstallFilter picks the packet filter — required, so the question is which, not // whether — and installs it. func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { filter, err := decide(Choice{ Name: "packet-filter", Question: "Which packet filter should this machine run?", Options: []string{"nftables"}, }, o.Answers["packet-filter"], o.Prompt, say) if err != nil { return "", err } if o.Adopted { // The firewall found here stays in force until the node converges; the filter is // chosen now and assigned by the flip (novox/hq ADR 0100). say(" not installed " + filter + " — this machine is adopted; converging it assigns " + filter) return filter, nil } return filter, InstallFromCatalogue(ctx, o, control, filter, say) } // InstallExtras installs what was asked for beyond the floor. // // One refusal per act: an extra that cannot be installed fails the run, because somebody asked // for it by name and a mesh that reports success minus one thing is reporting the wrong thing. func InstallExtras(ctx context.Context, o Options, control controlPlane, say func(string)) error { asked, err := decide(Choice{ Name: "extras", Question: "Anything beyond the floor? (comma-separated catalogue modules — " + "gitea, step-ca, dnsmasq — or nothing)", Default: "none", }, strings.Join(o.Extras, ","), o.Prompt, say) if err != nil { return err } if asked == "" || asked == "none" { say(" extras none") return nil } for _, extra := range strings.Split(asked, ",") { if extra = strings.TrimSpace(extra); extra == "" { continue } if err := InstallFromCatalogue(ctx, o, control, extra, say); err != nil { return fmt.Errorf("%s was asked for and could not be installed: %w", extra, err) } } return nil } // builds says whether a manifest declares anything to build. func builds(manifest []byte) bool { var m struct { Build *struct { Artifacts []json.RawMessage `json:"artifacts"` } `json:"build"` } if err := json.Unmarshal(manifest, &m); err != nil { return false } return m.Build != nil && len(m.Build.Artifacts) > 0 } // endpointAgrees holds the endpoint other machines dial to the port this node gave the private // network's hub (novox/hq ADR 0100): the hub binds what --hub-port says, so an endpoint naming // another port is an address nothing answers on. A host alone takes the hub's port. func endpointAgrees(endpoint string, hub int) (string, error) { _, portText, err := net.SplitHostPort(endpoint) var missing *net.AddrError if errors.As(err, &missing) && missing.Err == "missing port in address" { return net.JoinHostPort(strings.Trim(endpoint, "[]"), strconv.Itoa(hub)), nil } if err != nil { return "", fmt.Errorf("--endpoint %q is not host:port: %w", endpoint, err) } port, err := strconv.Atoi(portText) if err != nil { return "", fmt.Errorf("--endpoint %q does not end in a port", endpoint) } if port != hub { return "", fmt.Errorf("--endpoint %s names port %d and the private network's hub binds %d "+ "(--hub-port): other machines would dial a port nothing answers on. Give one port for the "+ "hub; nothing was changed", endpoint, port, hub) } return endpoint, nil } // derivedEndpoint is the default place other machines dial for the private network: the same host // they already dial for the broker, on WireGuard's ordinary port. One fact, not two. func derivedEndpoint(brokerAddress string, hub int) string { host, _, err := net.SplitHostPort(brokerAddress) if err != nil || host == "" { return "" } return net.JoinHostPort(host, strconv.Itoa(hub)) } func refOr(ref string) string { if ref == "" { return "main" } return ref }