Build the Go toolchain at genesis's base step, beside the base (hq issue 616)

Genesis's base step built only mesh-tools, the TypeScript toolchain. The store's and the bus's modules
carry Go bundles, compiled in mesh-tools-go, so genesis stopped at the store step: "this mesh holds no
copy of it. Build mesh-tools-go first". The base step now builds mesh-tools-go too, from
--tools-go-source or the repository beside --tools-source, and a test holds every toolchain a genesis
module's bundles compile in, read from the catalogue, to the ones the step builds.

cmd/mesh-bootstrap/main.go

@@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
 
   --tools-source     the repository the shared base is built from
   --tools-ref        what of it to build (default main)
+  --tools-go-source  the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)
+  --tools-go-ref     what of it to build (default main)
   --catalog-source   the catalogue REPOSITORY, for building its modules;
                      --catalog is the checkout that says what they are
   --catalog-ref      what of it to build (default main)
@@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
 		"the repository the shared base is built from")
 	set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
 		"what of it to build (default main)")
+	set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository,
+		"the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)")
+	set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref,
+		"what of it to build (default main)")
 	set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
 		"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
 	set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,

internal/bootstrap/bootstrap.go

@@ -174,6 +174,10 @@ type Options struct {
 	// ToolsSource is where the shared base is built from. Everything with code of its own
 	// compiles against it, so it is the first thing the mesh builds for itself.
 	ToolsSource Source
+	// ToolsGoSource is where the Go toolchain is built from (`mesh-tools-go`, ADR 0142): every Go
+	// bundle compiles in it, and genesis's own modules have Go code (issue 616). Empty means the
+	// repository beside ToolsSource named mesh-tools-go, as the mesh keeps it.
+	ToolsGoSource Source
 	// CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue
 	// CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it.
 	CatalogSource Source

internal/bootstrap/genesis_toolchains_test.go

@@ -0,0 +1,89 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Defends novox/hq issue 616: genesis's base step builds the toolchain of every language a module
+// genesis installs is compiled in. It built only mesh-tools, and the store's module, with Go code,
+// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first".
+
+// genesisModules is every catalogue module genesis installs on its own, read from the steps that
+// install them: the registry, the build agent, the store, the bus, the vault, the catalogue and the
+// filter (extras are the operator's choice and come after the base too, but are not genesis's own).
+var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "mesh-catalog", "nftables"}
+
+// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real
+// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a
+// genesis.
+func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) {
+	built := map[string]bool{}
+	for _, tc := range genesisToolchains {
+		built[tc.Language] = true
+	}
+	for _, module := range genesisModules {
+		var m struct {
+			Build struct {
+				Artifacts []struct {
+					Name, Kind, Language string
+				} `json:"artifacts"`
+			} `json:"build"`
+		}
+		if err := json.Unmarshal(theCataloguesModule(t, module), &m); err != nil {
+			t.Fatalf("%s: %v", module, err)
+		}
+		for _, a := range m.Build.Artifacts {
+			if a.Kind != "bundle" {
+				continue // an image builds on the bases its Dockerfile names, pinned upstream
+			}
+			if !built[a.Language] {
+				t.Errorf("%s's %q is a %s bundle, and genesis's base step builds no %s toolchain",
+					module, a.Name, a.Language, a.Language)
+			}
+		}
+	}
+}
+
+// And the base step asks for each toolchain's build: the base, then the Go toolchain beside it.
+func TestTheBaseStepBuildsTheGoToolchainBesideTheBase(t *testing.T) {
+	runtime := &asked{answer: aMeshThatAgrees(nil)}
+	control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
+	o := Options{ToolsSource: Source{Repository: "git://192.0.2.30/mesh-tools.git"}}
+	if err := BuildBase(context.Background(), o, control, quietly); err != nil {
+		t.Fatal(err)
+	}
+	var builds []string
+	for _, c := range runtime.commands {
+		if strings.Contains(c, "/mesh-controller build ") {
+			builds = append(builds, c[strings.Index(c, "build "):])
+		}
+	}
+	want := []string{
+		"build git://192.0.2.30/mesh-tools.git --ref main --wait 1200s",
+		"build git://192.0.2.30/mesh-tools-go.git --ref main --wait 1200s",
+	}
+	if strings.Join(builds, "\n") != strings.Join(want, "\n") {
+		t.Fatalf("the base step built\n%s\nand not\n%s", strings.Join(builds, "\n"), strings.Join(want, "\n"))
+	}
+}
+
+func TestTheGoToolchainIsAsGivenOrBesideTheBase(t *testing.T) {
+	for _, c := range []struct{ tools, given, want string }{
+		{"http://forge/novox/mesh-tools.git", "", "http://forge/novox/mesh-tools-go.git"},
+		{"http://forge/novox/mesh-tools", "", "http://forge/novox/mesh-tools-go"},
+		{"http://forge/other/base.git", "http://forge/other/go.git", "http://forge/other/go.git"},
+	} {
+		got, err := toolsGoSource(Options{ToolsSource: Source{Repository: c.tools}, ToolsGoSource: Source{Repository: c.given}})
+		if err != nil || got.Repository != c.want {
+			t.Errorf("tools %s, given %q: %q %v; want %s", c.tools, c.given, got.Repository, err, c.want)
+		}
+	}
+	if _, err := toolsGoSource(Options{ToolsSource: Source{Repository: "http://forge/other/base.git"}}); err == nil ||
+		!strings.Contains(err.Error(), "--tools-go-source") {
+		t.Errorf("a base not named mesh-tools, and no Go toolchain given, was not refused naming the flag: %v", err)
+	}
+}

internal/bootstrap/phase2.go

@@ -25,20 +25,62 @@ import (
 // buildWait bounds one module build. Generous, because the first build compiles a toolchain.
 const buildWait = 20 * time.Minute
 
-// BuildBase asks the mesh to build the shared base every module with code of its own stands on.
+// BuildBase asks the mesh to build the shared base every module with code of its own stands on, and
+// the toolchain of every language genesis's own modules are written in.
 //
 // **First, because until it exists nothing else with code can be built.** Not registered as a
 // module here: it is never assigned — it runs nowhere — and the build itself records what was
 // made, which is all anything downstream reads.
+//
+// **Every toolchain a genesis module compiles in** (novox/hq issue 616). A bundle is compiled in its
+// language's base (mesh-controller internal/builder toolchain.go): TypeScript in mesh-tools, Go in
+// mesh-tools-go. The store's and the bus's modules carry Go code, so a genesis that built only
+// mesh-tools stopped at the store step: "this mesh holds no copy of it. Build mesh-tools-go first".
+// The bases are the ones genesisToolchains names, and a test holds them to the languages of every
+// module genesis installs, read from the catalogue.
 func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error {
 	if o.ToolsSource.Repository == "" {
 		return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " +
 			"own repository, and an installer told nothing cannot know where that is")
 	}
-	say("  building          " + o.ToolsSource.Repository + " at " + refOr(o.ToolsSource.Ref))
-	_, err := control.within(buildWait).tell(ctx,
-		"build", o.ToolsSource.Repository, "--ref", refOr(o.ToolsSource.Ref), "--wait", "1200s")
-	return err
+	goSource, err := toolsGoSource(o)
+	if err != nil {
+		return err
+	}
+	for _, src := range []Source{o.ToolsSource, goSource} {
+		say("  building          " + src.Repository + " at " + refOr(src.Ref))
+		if _, err := control.within(buildWait).tell(ctx,
+			"build", src.Repository, "--ref", refOr(src.Ref), "--wait", "1200s"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// genesisToolchains is the base each language's bundles compile in, for the languages genesis's own
+// modules are written in — the same pairs as mesh-controller internal/builder toolchain.go — in the
+// order BuildBase builds them.
+var genesisToolchains = []struct{ Language, Base string }{
+	{"typescript", "mesh-tools"},
+	{"go", "mesh-tools-go"},
+}
+
+// toolsGoSource is where the Go toolchain is built from: as given, or the repository beside the base's
+// named mesh-tools-go, which is how the mesh keeps its repositories side by side.
+func toolsGoSource(o Options) (Source, error) {
+	if o.ToolsGoSource.Repository != "" {
+		return o.ToolsGoSource, nil
+	}
+	repo := strings.TrimSuffix(o.ToolsSource.Repository, "/")
+	suffix := ""
+	if strings.HasSuffix(repo, ".git") {
+		repo, suffix = strings.TrimSuffix(repo, ".git"), ".git"
+	}
+	if !strings.HasSuffix(repo, "mesh-tools") {
+		return Source{}, fmt.Errorf("the Go toolchain is built from mesh-tools-go, and --tools-source (%s) is not a "+
+			"repository named mesh-tools to find it beside; give --tools-go-source", o.ToolsSource.Repository)
+	}
+	return Source{Repository: repo + "-go" + suffix, Ref: o.ToolsGoSource.Ref}, nil
 }
 
 // InstallFromCatalogue builds a catalogue module and installs it on this machine.