diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 4bc7517..369a744 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh --tools-source the repository the shared base is built from --tools-ref what of it to build (default main) + --tools-go-source the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source) + --tools-go-ref what of it to build (default main) --catalog-source the catalogue REPOSITORY, for building its modules; --catalog is the checkout that says what they are --catalog-ref what of it to build (default main) @@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { "the repository the shared base is built from") set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref, "what of it to build (default main)") + set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository, + "the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)") + set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref, + "what of it to build (default main)") set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository, "the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are") set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref, diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index c047eb1..dbb2e08 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -174,6 +174,10 @@ type Options struct { // ToolsSource is where the shared base is built from. Everything with code of its own // compiles against it, so it is the first thing the mesh builds for itself. ToolsSource Source + // ToolsGoSource is where the Go toolchain is built from (`mesh-tools-go`, ADR 0142): every Go + // bundle compiles in it, and genesis's own modules have Go code (issue 616). Empty means the + // repository beside ToolsSource named mesh-tools-go, as the mesh keeps it. + ToolsGoSource Source // CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue // CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it. CatalogSource Source diff --git a/internal/bootstrap/genesis_toolchains_test.go b/internal/bootstrap/genesis_toolchains_test.go new file mode 100644 index 0000000..8523054 --- /dev/null +++ b/internal/bootstrap/genesis_toolchains_test.go @@ -0,0 +1,89 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +// Defends novox/hq issue 616: genesis's base step builds the toolchain of every language a module +// genesis installs is compiled in. It built only mesh-tools, and the store's module, with Go code, +// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first". + +// genesisModules is every catalogue module genesis installs on its own, read from the steps that +// install them: the registry, the build agent, the store, the bus, the vault, the catalogue and the +// filter (extras are the operator's choice and come after the base too, but are not genesis's own). +var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "mesh-catalog", "nftables"} + +// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real +// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a +// genesis. +func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) { + built := map[string]bool{} + for _, tc := range genesisToolchains { + built[tc.Language] = true + } + for _, module := range genesisModules { + var m struct { + Build struct { + Artifacts []struct { + Name, Kind, Language string + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(theCataloguesModule(t, module), &m); err != nil { + t.Fatalf("%s: %v", module, err) + } + for _, a := range m.Build.Artifacts { + if a.Kind != "bundle" { + continue // an image builds on the bases its Dockerfile names, pinned upstream + } + if !built[a.Language] { + t.Errorf("%s's %q is a %s bundle, and genesis's base step builds no %s toolchain", + module, a.Name, a.Language, a.Language) + } + } + } +} + +// And the base step asks for each toolchain's build: the base, then the Go toolchain beside it. +func TestTheBaseStepBuildsTheGoToolchainBesideTheBase(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + o := Options{ToolsSource: Source{Repository: "git://192.0.2.30/mesh-tools.git"}} + if err := BuildBase(context.Background(), o, control, quietly); err != nil { + t.Fatal(err) + } + var builds []string + for _, c := range runtime.commands { + if strings.Contains(c, "/mesh-controller build ") { + builds = append(builds, c[strings.Index(c, "build "):]) + } + } + want := []string{ + "build git://192.0.2.30/mesh-tools.git --ref main --wait 1200s", + "build git://192.0.2.30/mesh-tools-go.git --ref main --wait 1200s", + } + if strings.Join(builds, "\n") != strings.Join(want, "\n") { + t.Fatalf("the base step built\n%s\nand not\n%s", strings.Join(builds, "\n"), strings.Join(want, "\n")) + } +} + +func TestTheGoToolchainIsAsGivenOrBesideTheBase(t *testing.T) { + for _, c := range []struct{ tools, given, want string }{ + {"http://forge/novox/mesh-tools.git", "", "http://forge/novox/mesh-tools-go.git"}, + {"http://forge/novox/mesh-tools", "", "http://forge/novox/mesh-tools-go"}, + {"http://forge/other/base.git", "http://forge/other/go.git", "http://forge/other/go.git"}, + } { + got, err := toolsGoSource(Options{ToolsSource: Source{Repository: c.tools}, ToolsGoSource: Source{Repository: c.given}}) + if err != nil || got.Repository != c.want { + t.Errorf("tools %s, given %q: %q %v; want %s", c.tools, c.given, got.Repository, err, c.want) + } + } + if _, err := toolsGoSource(Options{ToolsSource: Source{Repository: "http://forge/other/base.git"}}); err == nil || + !strings.Contains(err.Error(), "--tools-go-source") { + t.Errorf("a base not named mesh-tools, and no Go toolchain given, was not refused naming the flag: %v", err) + } +} diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index 615191a..e49eb1b 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -25,20 +25,62 @@ import ( // buildWait bounds one module build. Generous, because the first build compiles a toolchain. const buildWait = 20 * time.Minute -// BuildBase asks the mesh to build the shared base every module with code of its own stands on. +// BuildBase asks the mesh to build the shared base every module with code of its own stands on, and +// the toolchain of every language genesis's own modules are written in. // // **First, because until it exists nothing else with code can be built.** Not registered as a // module here: it is never assigned — it runs nowhere — and the build itself records what was // made, which is all anything downstream reads. +// +// **Every toolchain a genesis module compiles in** (novox/hq issue 616). A bundle is compiled in its +// language's base (mesh-controller internal/builder toolchain.go): TypeScript in mesh-tools, Go in +// mesh-tools-go. The store's and the bus's modules carry Go code, so a genesis that built only +// mesh-tools stopped at the store step: "this mesh holds no copy of it. Build mesh-tools-go first". +// The bases are the ones genesisToolchains names, and a test holds them to the languages of every +// module genesis installs, read from the catalogue. func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error { if o.ToolsSource.Repository == "" { return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " + "own repository, and an installer told nothing cannot know where that is") } - say(" building " + o.ToolsSource.Repository + " at " + refOr(o.ToolsSource.Ref)) - _, err := control.within(buildWait).tell(ctx, - "build", o.ToolsSource.Repository, "--ref", refOr(o.ToolsSource.Ref), "--wait", "1200s") - return err + goSource, err := toolsGoSource(o) + if err != nil { + return err + } + for _, src := range []Source{o.ToolsSource, goSource} { + say(" building " + src.Repository + " at " + refOr(src.Ref)) + if _, err := control.within(buildWait).tell(ctx, + "build", src.Repository, "--ref", refOr(src.Ref), "--wait", "1200s"); err != nil { + return err + } + } + return nil +} + +// genesisToolchains is the base each language's bundles compile in, for the languages genesis's own +// modules are written in — the same pairs as mesh-controller internal/builder toolchain.go — in the +// order BuildBase builds them. +var genesisToolchains = []struct{ Language, Base string }{ + {"typescript", "mesh-tools"}, + {"go", "mesh-tools-go"}, +} + +// toolsGoSource is where the Go toolchain is built from: as given, or the repository beside the base's +// named mesh-tools-go, which is how the mesh keeps its repositories side by side. +func toolsGoSource(o Options) (Source, error) { + if o.ToolsGoSource.Repository != "" { + return o.ToolsGoSource, nil + } + repo := strings.TrimSuffix(o.ToolsSource.Repository, "/") + suffix := "" + if strings.HasSuffix(repo, ".git") { + repo, suffix = strings.TrimSuffix(repo, ".git"), ".git" + } + if !strings.HasSuffix(repo, "mesh-tools") { + return Source{}, fmt.Errorf("the Go toolchain is built from mesh-tools-go, and --tools-source (%s) is not a "+ + "repository named mesh-tools to find it beside; give --tools-go-source", o.ToolsSource.Repository) + } + return Source{Repository: repo + "-go" + suffix, Ref: o.ToolsGoSource.Ref}, nil } // InstallFromCatalogue builds a catalogue module and installs it on this machine.