cmd/mesh-bootstrap/main.go

@@ -117,6 +117,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
   --catalog-source   the catalogue REPOSITORY, for building its modules;
                      --catalog is the checkout that says what they are
   --catalog-ref      what of it to build (default main)
+  --sdk-source       the repository the shared library is built from
+  --sdk-ref          what of it to build (default main)
   --private-network  which private network to run (wireguard)
   --endpoint         host:port other machines dial for it; derived from the
                      broker address when unsaid
@@ -248,6 +250,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
 		"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
 	set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
 		"what of it to build (default main)")
+	set.StringVar(&opts.SDKSource.Repository, "sdk-source", opts.SDKSource.Repository,
+		"the repository the shared library is built from, published before the base resolves it")
+	set.StringVar(&opts.SDKSource.Ref, "sdk-ref", opts.SDKSource.Ref,
+		"what of it to build (default main)")
 	set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network")
 	if opts.Answers == nil {
 		opts.Answers = map[string]string{}

internal/bootstrap/bootstrap.go

@@ -53,6 +53,8 @@ const (
 	StepControlPlane Step = "control-plane"
 	StepRetire       Step = "retire"
 	StepBuilder      Step = "builder"
+	StepPackages     Step = "packages"
+	StepSDK          Step = "sdk"
 	StepBase         Step = "base"
 	StepStore        Step = "store"
 	StepCatalogue    Step = "catalogue"
@@ -75,6 +77,7 @@ const (
 var Steps = []Step{
 	StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify,
 	StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder,
+	StepPackages, StepSDK,
 	// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
 	// build, to say what it holds, on its network, filtering. They used to be things somebody
 	// typed afterwards, which is how they went missing without anything complaining.
@@ -164,6 +167,9 @@ type Options struct {
 	// CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue
 	// CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it.
 	CatalogSource Source
+	// SDKSource is where the mesh's shared library is built from. It is published to the package
+	// registry before the base is built, because the base resolves it by version (novox/hq ADR 0076).
+	SDKSource Source
 	// Site is where this machine sits, for the private network's placement.
 	Site string
 	// Answers are the choices, answered by flag: name → answer. What a terminal would be asked.
@@ -563,6 +569,18 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 		return result, failed(StepBuilder, err)
 	}
 
+	// ---- packages — the registry, before the base that resolves the SDK from it ----------
+	say("packages — a registry answers, and the SDK is in it, before the base is built")
+	if err := RaisePackageRegistry(ctx, o, d, permanentControl, say); err != nil {
+		return result, failed(StepPackages, err)
+	}
+
+	// ---- sdk — published on a public base, so this needs no toolchain --------------------
+	say("sdk — the shared library, published by version so the base can resolve it")
+	if err := PublishTheSDK(ctx, o, permanentControl, say); err != nil {
+		return result, failed(StepSDK, err)
+	}
+
 	// ---- 13. base -------------------------------------------------------------------------
 	say("base — the shared toolchain and runtime everything with code stands on")
 	if err := BuildBase(ctx, o, permanentControl, say); err != nil {

internal/bootstrap/phase_packages.go

@@ -0,0 +1,249 @@
+package bootstrap
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Raising the package registry, before the base is built.
+//
+// The base (mesh-tools) resolves the SDK by version from the mesh's package registry rather than
+// cloning it from a git URL (novox/hq ADR 0076, issue 053). So the registry has to answer, and the
+// SDK has to be in it, before the base build runs. That is a pivot like the control plane's: gitea's
+// SERVER is raised directly here, on the substrate's own postgres, and adopted as an ordinary module
+// only after the base exists (which is what lets its provisioner image — built on the base — run).
+//
+// Nothing here is the steady state. It is the smallest set of acts that puts a working npm registry
+// in front of the base build: a database, a server, an admin, one org, the builder's own account,
+// and the SDK published under it. The gitea MODULE, installed after the base, takes all of this over.
+
+const (
+	// substrateStore is the substrate's postgres container — the mesh's own memory, raised from the
+	// bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051).
+	substrateStore = "mesh-store"
+	// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module.
+	giteaBootstrap = "mesh-gitea-server"
+	// giteaImage is the same upstream image the gitea module runs, pinned identically so the module
+	// adopts the running server rather than replacing it.
+	giteaImage = "gitea/gitea@sha256:dfc61e347c8b582df918f4556401bf2cecdfbdb56c5282ae9488dd76fca3e41c"
+	// packagesOrg is the npm owner: every module consumes `@novox/*` from this gitea org.
+	packagesOrg = "novox"
+	// packagesTeam is the org team whose members may read and write the org's packages.
+	packagesTeam = "packages"
+	// giteaAdminUser is the admin the bootstrap creates and the provisioner later authenticates as.
+	giteaAdminUser = "mesh-admin"
+	// builderGiteaUser is the gitea account the builder publishes and pulls with at genesis. It is
+	// the `as` the builder's static package binding names.
+	builderGiteaUser = "mesh-builder"
+	// giteaDBRole/giteaDBName is gitea's own database in the substrate store.
+	giteaDBRole = "mesh_gitea"
+	giteaDBName = "mesh_gitea"
+	// giteaPort is where the raised server answers on the machine.
+	giteaPort = 3000
+)
+
+// RaisePackageRegistry puts a working npm registry in front of the base build. It is idempotent:
+// every step tolerates having been done, because genesis is safe to run again.
+func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control controlPlane,
+	say func(string)) error {
+	run := d.Run
+
+	// The passwords the mesh mints for this pivot. gitea's database password and its admin password
+	// are the mesh's, generated here; the builder's is generated and also becomes its own-secret.
+	dbPassword := newPassword()
+	adminPassword := newPassword()
+	builderPassword := newPassword()
+
+	say("  seeding gitea's database in the substrate store")
+	if err := seedGiteaDatabase(ctx, run, o.Timeout, dbPassword, say); err != nil {
+		return err
+	}
+
+	say("  raising the gitea server on that database")
+	if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, say); err != nil {
+		return err
+	}
+
+	say("  waiting for gitea to answer")
+	base := fmt.Sprintf("http://127.0.0.1:%d", giteaPort)
+	if err := waitForGitea(ctx, d, o, base, say); err != nil {
+		return err
+	}
+
+	say("  creating the gitea admin")
+	if err := createGiteaAdmin(ctx, run, o.Timeout, adminPassword, say); err != nil {
+		return err
+	}
+
+	admin := &giteaAdmin{base: base, user: giteaAdminUser, password: adminPassword,
+		client: &http.Client{Timeout: o.Timeout}}
+
+	say("  ensuring the npm org, its package team, and the builder's account")
+	if err := admin.ensureOrg(ctx, packagesOrg); err != nil {
+		return err
+	}
+	teamID, err := admin.ensureTeam(ctx, packagesOrg, packagesTeam)
+	if err != nil {
+		return err
+	}
+	if err := admin.ensureUser(ctx, builderGiteaUser, builderPassword); err != nil {
+		return err
+	}
+	if err := admin.addToTeam(ctx, teamID, builderGiteaUser); err != nil {
+		return err
+	}
+
+	say("  delivering the builder its registry credential")
+	if err := deliverBuilderNpm(ctx, o, control, builderPassword, say); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// seedGiteaDatabase creates gitea's role and database inside the substrate postgres, the same way
+// the substrate creates its own — psql run through the store container (the map's Route B). The role
+// is created before the database because the database is owned by it. Both are tolerant of already
+// existing, so a re-run changes nothing.
+func seedGiteaDatabase(ctx context.Context, run Runner, timeout time.Duration, password string,
+	say func(string)) error {
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	// A single transaction-free script: CREATE ROLE/DATABASE cannot run inside one, and DO blocks
+	// let "already there" be silent rather than an error the caller must parse.
+	script := fmt.Sprintf(`
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '%[1]s') THEN
+    CREATE ROLE %[1]s LOGIN PASSWORD '%[2]s';
+  ELSE
+    ALTER ROLE %[1]s LOGIN PASSWORD '%[2]s';
+  END IF;
+END $$;
+SELECT 'CREATE DATABASE %[3]s OWNER %[1]s'
+  WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%[3]s')\gexec
+`, giteaDBRole, password, giteaDBName)
+
+	if _, err := run(asking, "docker", "exec", "-i", substrateStore,
+		"psql", "-U", "postgres", "-v", "ON_ERROR_STOP=1", "-c", script); err != nil {
+		return fmt.Errorf("could not seed gitea's database in %s: %w", substrateStore, err)
+	}
+	return nil
+}
+
+// raiseGiteaServer starts the gitea server container against the substrate store. It joins the
+// store's network namespace so `127.0.0.1:5432` reaches postgres, and publishes its own port on the
+// machine so the builder and this installer can reach it. Started if absent, left alone if present.
+func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, dbPassword string,
+	say func(string)) error {
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	// Already there: a re-run does not raise a second one. `docker start` is a no-op on a running
+	// container and revives a stopped one.
+	if out, _ := run(asking, "docker", "inspect", "--format", "{{.Id}}", giteaBootstrap); strings.TrimSpace(out) != "" {
+		_, _ = run(asking, "docker", "start", giteaBootstrap)
+		return nil
+	}
+
+	env := []string{
+		"-e", "GITEA__database__DB_TYPE=postgres",
+		// The store is reached on the shared network namespace's loopback.
+		"-e", "GITEA__database__HOST=127.0.0.1:5432",
+		"-e", "GITEA__database__NAME=" + giteaDBName,
+		"-e", "GITEA__database__USER=" + giteaDBRole,
+		"-e", "GITEA__database__PASSWD=" + dbPassword,
+		// Skip the install wizard: the mesh configures gitea, not a person at a browser.
+		"-e", "GITEA__security__INSTALL_LOCK=true",
+		"-e", "USER_UID=1000", "-e", "USER_GID=1000",
+	}
+	args := append([]string{
+		"run", "-d", "--name", giteaBootstrap,
+		"--network", "container:" + substrateStore,
+		"--restart", "unless-stopped",
+	}, env...)
+	args = append(args, giteaImage)
+
+	if _, err := run(asking, "docker", args...); err != nil {
+		return fmt.Errorf("could not raise the gitea server: %w", err)
+	}
+	return nil
+}
+
+// waitForGitea polls gitea's version endpoint until it answers or the wait runs out. A container
+// that is up is not a forge that serves; `/api/v1/version` is the question whose answer means it is.
+func waitForGitea(ctx context.Context, d Deps, o Options, base string, say func(string)) error {
+	deadline := time.Now().Add(o.Wait)
+	url := base + "/api/v1/version"
+	for {
+		status, _, err := d.Fetch(ctx, url)
+		if err == nil && status == http.StatusOK {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("gitea did not answer at %s within %s", url, o.Wait)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}
+
+// createGiteaAdmin creates the mesh's gitea admin through the server's own CLI. Tolerant of the
+// admin already existing, because a re-run must not fail on it — and it resets the password every
+// run, so a rotated admin secret takes.
+func createGiteaAdmin(ctx context.Context, run Runner, timeout time.Duration, password string,
+	say func(string)) error {
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	// Create, tolerating "already exists"; then set the password unconditionally so a re-run
+	// converges. Run as the gitea user, which owns the data the CLI reads.
+	create := fmt.Sprintf(
+		"gitea admin user create --admin --username %s --email %s@localhost --password %q --must-change-password=false || true; "+
+			"gitea admin user change-password --username %s --password %q || true",
+		giteaAdminUser, giteaAdminUser, password, giteaAdminUser, password)
+	if _, err := run(asking, "docker", "exec", "-u", "git", giteaBootstrap,
+		"sh", "-c", create); err != nil {
+		return fmt.Errorf("could not create the gitea admin: %w", err)
+	}
+	return nil
+}
+
+// deliverBuilderNpm seals the builder's registry password to this node as its `npm-password`
+// own-secret, the same way the control plane's store connections are delivered — carry the value in,
+// `secret accept`, and the next push writes it sealed where the builder reads it.
+func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string,
+	say func(string)) error {
+	at := "/accepting-npm-password"
+	if err := control.carrying(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil {
+		return err
+	}
+	if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil {
+		return err
+	}
+	// Push so the sealed secret reaches the builder, which restarts on it and comes back credentialed.
+	if _, err := control.tell(ctx, "push", o.Node); err != nil {
+		return err
+	}
+	return nil
+}
+
+// PublishTheSDK dispatches a build of the SDK to the builder. The builder has its registry
+// credential by now, so the build's `npm publish` authenticates; the artifact is a `package`, built
+// on a public base, so this needs no toolchain — which is the whole point of doing it before the base.
+func PublishTheSDK(ctx context.Context, o Options, control controlPlane, say func(string)) error {
+	if o.SDKSource.Repository == "" {
+		return fmt.Errorf("raising the registry needs --sdk-source: the SDK is built from its own " +
+			"repository, and an installer told nothing cannot know where that is")
+	}
+	say("  publishing " + o.SDKSource.Repository + " at " + refOr(o.SDKSource.Ref))
+	_, err := control.within(buildWait).tell(ctx,
+		"build", o.SDKSource.Repository, "--ref", refOr(o.SDKSource.Ref), "--wait", "1200s")
+	return err
+}

internal/bootstrap/phase_packages_gitea.go

@@ -0,0 +1,171 @@
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// A minimal gitea admin client, for the genesis pivot only. The gitea MODULE carries the real one
+// (its TS provisioner); this exists because at genesis that module cannot be built yet — its image
+// stands on the base, which is what this is helping to build. It does the few acts the pivot needs
+// and nothing more: an org, a team, a user, a membership. Everything is idempotent, because genesis
+// is safe to run again.
+type giteaAdmin struct {
+	base     string
+	user     string
+	password string
+	client   *http.Client
+}
+
+func (g *giteaAdmin) do(ctx context.Context, method, path string, body any) (int, []byte, error) {
+	var payload io.Reader
+	if body != nil {
+		raw, err := json.Marshal(body)
+		if err != nil {
+			return 0, nil, err
+		}
+		payload = bytes.NewReader(raw)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, g.base+"/api/v1"+path, payload)
+	if err != nil {
+		return 0, nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(g.user+":"+g.password)))
+	res, err := g.client.Do(req)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer res.Body.Close()
+	out, _ := io.ReadAll(res.Body)
+	return res.StatusCode, out, nil
+}
+
+// ok reports whether a status is one this pivot treats as success — the create succeeded, or the
+// thing already exists (422/409), which for an idempotent step is the same outcome.
+func ensured(status int) bool {
+	return status/100 == 2 || status == http.StatusUnprocessableEntity || status == http.StatusConflict
+}
+
+func (g *giteaAdmin) ensureOrg(ctx context.Context, name string) error {
+	status, body, err := g.do(ctx, http.MethodPost, "/orgs",
+		map[string]any{"username": name, "visibility": "private"})
+	if err != nil {
+		return err
+	}
+	if !ensured(status) {
+		return fmt.Errorf("could not create the gitea org %q: %d %s", name, status, body)
+	}
+	return nil
+}
+
+// ensureTeam creates the org's package team with write on packages and returns its id, finding the
+// existing one when a create loses to a concurrent one.
+func (g *giteaAdmin) ensureTeam(ctx context.Context, org, team string) (int, error) {
+	if id, err := g.findTeam(ctx, org, team); err != nil {
+		return 0, err
+	} else if id != 0 {
+		return id, nil
+	}
+	status, body, err := g.do(ctx, http.MethodPost, "/orgs/"+org+"/teams", map[string]any{
+		"name":                      team,
+		"permission":                "read",
+		"units_map":                 map[string]string{"repo.packages": "write"},
+		"includes_all_repositories": true,
+		"can_create_org_repo":       false,
+	})
+	if err != nil {
+		return 0, err
+	}
+	if status/100 == 2 {
+		var made struct {
+			ID int `json:"id"`
+		}
+		if err := json.Unmarshal(body, &made); err == nil && made.ID != 0 {
+			return made.ID, nil
+		}
+	}
+	// A lost race, or a body without an id: re-find.
+	if id, err := g.findTeam(ctx, org, team); err == nil && id != 0 {
+		return id, nil
+	}
+	return 0, fmt.Errorf("could not create the gitea team %q in %q: %d %s", team, org, status, body)
+}
+
+func (g *giteaAdmin) findTeam(ctx context.Context, org, team string) (int, error) {
+	status, body, err := g.do(ctx, http.MethodGet, "/orgs/"+org+"/teams?limit=50", nil)
+	if err != nil {
+		return 0, err
+	}
+	if status != http.StatusOK {
+		return 0, nil
+	}
+	var teams []struct {
+		ID   int    `json:"id"`
+		Name string `json:"name"`
+	}
+	if err := json.Unmarshal(body, &teams); err != nil {
+		return 0, err
+	}
+	for _, t := range teams {
+		if t.Name == team {
+			return t.ID, nil
+		}
+	}
+	return 0, nil
+}
+
+// ensureUser creates a gitea user with the mesh's minted password, or resets that user's password
+// when it already exists, so a rotation takes.
+func (g *giteaAdmin) ensureUser(ctx context.Context, name, password string) error {
+	status, body, err := g.do(ctx, http.MethodPost, "/admin/users", map[string]any{
+		"username":             name,
+		"email":                name + "@localhost",
+		"password":             password,
+		"must_change_password": false,
+	})
+	if err != nil {
+		return err
+	}
+	if status/100 == 2 {
+		return nil
+	}
+	if status == http.StatusUnprocessableEntity || status == http.StatusConflict {
+		// Already there: reset the password so this run's credential is the one that works.
+		reset, rbody, err := g.do(ctx, http.MethodPatch, "/admin/users/"+name,
+			map[string]any{"login_name": name, "password": password, "must_change_password": false})
+		if err != nil {
+			return err
+		}
+		if reset/100 == 2 {
+			return nil
+		}
+		return fmt.Errorf("could not reset the gitea user %q: %d %s", name, reset, rbody)
+	}
+	return fmt.Errorf("could not create the gitea user %q: %d %s", name, status, body)
+}
+
+func (g *giteaAdmin) addToTeam(ctx context.Context, teamID int, user string) error {
+	status, body, err := g.do(ctx, http.MethodPut, fmt.Sprintf("/teams/%d/members/%s", teamID, user), nil)
+	if err != nil {
+		return err
+	}
+	if !ensured(status) {
+		return fmt.Errorf("could not add %q to team %d: %d %s", user, teamID, status, body)
+	}
+	return nil
+}
+
+// newPassword is a mesh-minted secret: 32 bytes of randomness, URL-safe so it survives a connection
+// string and an .npmrc without escaping.
+func newPassword() string {
+	b := make([]byte, 32)
+	_, _ = rand.Read(b)
+	return base64.RawURLEncoding.EncodeToString(b)
+}