Have genesis trust its registry before pushing to it (issue 92)

Genesis pushed the control plane's image to the mesh's plain-HTTP registry
before anything told the container runtime to trust it; the docker module
that writes that trust is applied long after. Genesis now writes the trust
itself at the registry step, with the same into-json writer and a reload,
for the address given and the node's mesh name on the node's registry
port, and refuses before any push unless the runtime says it trusts each.
The docker module later finds the mesh name there and takes it over
without a change.

internal/bootstrap/bootstrap.go

@@ -292,12 +292,14 @@ type Result struct {
 	Agent     string `json:"host-agent,omitempty"`
 
 	// Registry is the mesh's own artifact store, once it answers.
-	Registry         string `json:"registry,omitempty"`
-	RegistryReplied  int    `json:"registry-replied,omitempty"`
-	RegistryKnown    bool   `json:"registry-already-registered,omitempty"`
-	RegistryRunning  string `json:"registry-container,omitempty"`
-	PublishedAs      string `json:"control-plane-image,omitempty"`
-	PublishedAlready bool   `json:"control-plane-image-already-published,omitempty"`
+	Registry        string `json:"registry,omitempty"`
+	RegistryReplied int    `json:"registry-replied,omitempty"`
+	RegistryKnown   bool   `json:"registry-already-registered,omitempty"`
+	RegistryRunning string `json:"registry-container,omitempty"`
+	// RegistryTrusted are the names the runtime was told to trust the registry by, before the push.
+	RegistryTrusted  []string `json:"registry-trusted,omitempty"`
+	PublishedAs      string   `json:"control-plane-image,omitempty"`
+	PublishedAlready bool     `json:"control-plane-image-already-published,omitempty"`
 
 	// Permanent is the control plane as an ordinary module.
 	Permanent          string   `json:"permanent-container,omitempty"`
@@ -555,7 +557,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 			// Named rather than attempted. Everything from step 6 on is a conversation with a
 			// control plane that a dry run has not raised, so there is nothing to ask and nothing
 			// honest to report about the answers.
-			say("  would then        enrol " + o.Node + ", install the registry from " +
+			say("  would then        enrol " + o.Node + ", have the runtime trust the registry as " +
+				strings.Join(RegistryNames(o), ", ") + ",")
+			say("                    install the registry from " +
 				o.Catalogue + ", push the control plane's image into it,")
 			say("                    reinstall the control plane as a module, and drop " +
 				rewritten.TempName)
@@ -656,6 +660,13 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
 
 	// ---- 7. registry ----------------------------------------------------------------------
 	say("registry — somewhere for this mesh to keep its own images")
+	// Trusted by the runtime before anything is pushed to it, by every name the mesh will use for
+	// it here: the module that writes this trust is applied long after the push (novox/hq issue 92).
+	trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say)
+	result.RegistryTrusted = trusted
+	if err != nil {
+		return result, failed(StepRegistry, err)
+	}
 	registry, err := InstallRegistry(ctx, o, d, temporary, say)
 	result.Registry, result.RegistryRunning = registry.Address, registry.Container
 	result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered

internal/bootstrap/trust.go

@@ -0,0 +1,232 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92).
+//
+// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private
+// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry
+// it has not been told to trust, and the module that tells it — `docker`, which writes the
+// artifact store's reach into the runtime's configuration — is applied long after genesis pushes
+// the control plane's image. So genesis writes that trust itself, first, with the same writer the
+// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and
+// never restarted. It then asks the runtime whether it will speak to every name, and refuses with
+// the runtime's answer before anything is pushed.
+//
+// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation
+// could hold it, because the retire step re-applies the bundle and takes away a carried record the
+// bundle does not name. The `docker` module, applied later, declares the artifact store's reach —
+// the mesh name on the node's port — finds it already in the file it has no record of, and takes it
+// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). So nothing
+// changes on the machine at the handover, no reload; from then on the module owns that member. The
+// address the operator gave, when it is not the mesh name, stays as part of the machine's own
+// configuration, as the hand-written trust it replaces would have.
+
+// runtimeConfig is the container runtime's configuration, which the trust is written into. A
+// variable so a test writes into a file of its own.
+var runtimeConfig = "/etc/docker/daemon.json"
+
+// runtimeUnit is the runtime's service, reloaded when the file changed.
+const runtimeUnit = "docker.service"
+
+// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable
+// and default the controller composes `<node>.<suffix>` from (mesh-controller internal/overlay),
+// which is the name the artifact store's reach — and so every push by the mesh's builder — uses.
+const (
+	internalSuffixVar     = "MESH_INTERNAL_SUFFIX"
+	defaultInternalSuffix = "internal"
+)
+
+// RegistryNames are the names the mesh will use for its registry on this node, each as host:port.
+//
+// The address the operator gave, and the node's mesh name: the controller's reach for the artifact
+// store, which the builder pushes to without being told. Both on the port the registry module
+// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A
+// loopback name is left out: the runtime trusts loopback without being told, and the mesh never
+// names it to another node.
+func RegistryNames(o Options) []string {
+	port := strconv.Itoa(o.Ports.orDefaults().Registry)
+	var names []string
+	add := func(hostport string) {
+		host, _, err := net.SplitHostPort(hostport)
+		if err != nil || host == "" || isLoopback(host) {
+			return
+		}
+		for _, n := range names {
+			if n == hostport {
+				return
+			}
+		}
+		names = append(names, hostport)
+	}
+	if host, _, err := net.SplitHostPort(o.Registry); err == nil {
+		add(net.JoinHostPort(strings.Trim(host, "[]"), port))
+	}
+	if node := strings.TrimSpace(o.Node); node != "" {
+		add(net.JoinHostPort(node+"."+internalSuffix(), port))
+	}
+	return names
+}
+
+func internalSuffix() string {
+	if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" {
+		return strings.TrimPrefix(v, ".")
+	}
+	return defaultInternalSuffix
+}
+
+func isLoopback(host string) bool {
+	if strings.EqualFold(host, "localhost") {
+		return true
+	}
+	ip := net.ParseIP(strings.Trim(host, "[]"))
+	return ip != nil && ip.IsLoopback()
+}
+
+// trustDeclaration is the declaration genesis applies to write the trust: the same two resources
+// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it.
+func trustDeclaration(names []string) (*declaration.Declaration, error) {
+	keys, err := json.Marshal(map[string][]string{"insecure-registries": names})
+	if err != nil {
+		return nil, err
+	}
+	raw, err := json.Marshal(map[string]any{
+		"declaration": 1,
+		"resources": []map[string]any{
+			{"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644",
+				"into": "json", "content": string(keys) + "\n"},
+			{"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit,
+				"state": "running", "reload-on": []string{"genesis.registry-trust"}},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return declaration.ParseTrusted(raw)
+}
+
+// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here,
+// then refuses unless the runtime says it will speak plain HTTP to each.
+func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner,
+	say func(string)) ([]string, error) {
+
+	names := RegistryNames(o)
+	if len(names) == 0 {
+		say("  trusted           only loopback names this registry, which the runtime trusts already")
+		return nil, nil
+	}
+	d, err := trustDeclaration(names)
+	if err != nil {
+		return names, err
+	}
+
+	// One apply at a time on this machine, as ApplyBundle does: a host already running here may be
+	// writing the same file.
+	unlock, err := store.Lock(o.State, func() { say("  waiting           another apply holds this node's state") })
+	if err != nil {
+		return names, err
+	}
+	report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run,
+		func(line string) { say("  " + strings.TrimPrefix(line, "  ")) }, refuseSealed)
+	unlock()
+	if err != nil {
+		return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+
+			"The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+
+			"runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err)
+	}
+	if report.Changed() {
+		say("  trusted           " + strings.Join(names, ", ") + " — written into " + runtimeConfig +
+			", the runtime reloaded")
+	} else {
+		say("  trusted           " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so")
+	}
+	return names, runtimeWillSpeakTo(ctx, o, run, names)
+}
+
+// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out.
+//
+// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on
+// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error.
+func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error {
+	deadline := time.Now().Add(o.Wait)
+	var missing []string
+	var last error
+	for {
+		asking, cancel := context.WithTimeout(ctx, o.Timeout)
+		out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}")
+		cancel()
+		last = err
+		if err == nil {
+			missing, last = untrusted(out, names)
+			if last == nil && len(missing) == 0 {
+				return nil
+			}
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(answerEvery):
+		}
+	}
+	if last != nil {
+		return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last)
+	}
+	return fmt.Errorf(
+		"the container runtime does not trust %s after %s says it should, so it would refuse to push to "+
+			"it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+
+			"with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+
+			"Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit)
+}
+
+// untrusted reads the runtime's registry configuration and gives back the names it would speak
+// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in
+// a range the runtime trusts.
+func untrusted(registryConfig string, names []string) ([]string, error) {
+	var c struct {
+		InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"`
+		IndexConfigs          map[string]struct {
+			Secure bool `json:"Secure"`
+		} `json:"IndexConfigs"`
+	}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil {
+		return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err)
+	}
+	var missing []string
+	for _, name := range names {
+		if index, ok := c.IndexConfigs[name]; ok && !index.Secure {
+			continue
+		}
+		host, _, _ := net.SplitHostPort(name)
+		if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) {
+			continue
+		}
+		missing = append(missing, name)
+	}
+	return missing, nil
+}
+
+func inAny(ip net.IP, cidrs []string) bool {
+	for _, cidr := range cidrs {
+		if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}

internal/bootstrap/trust_test.go

@@ -0,0 +1,220 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
+// told the container runtime to trust that registry, so the push failed with "server gave HTTP
+// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
+// and the trust the mesh wrote later named the registry's default port rather than the node's.
+
+// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
+// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
+type aRuntime struct {
+	path     string
+	trusted  []string
+	commands []string
+	// ignoresReload is a runtime started with options of its own, which a reload does not change.
+	ignoresReload bool
+}
+
+func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
+	line := name + " " + strings.Join(args, " ")
+	r.commands = append(r.commands, line)
+	switch {
+	case name == "docker" && len(args) > 0 && args[0] == "info":
+		index := map[string]map[string]any{"docker.io": {"Secure": true}}
+		for _, n := range r.trusted {
+			index[n] = map[string]any{"Name": n, "Secure": false}
+		}
+		out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
+		return string(out), nil
+	case name == "systemctl" && strings.Contains(line, "reload"):
+		if !r.ignoresReload {
+			r.trusted = r.trusted[:0]
+			raw, _ := os.ReadFile(r.path)
+			var c struct {
+				Insecure []string `json:"insecure-registries"`
+			}
+			_ = json.Unmarshal(raw, &c)
+			r.trusted = append(r.trusted, c.Insecure...)
+		}
+	case strings.Contains(line, "is-enabled"):
+		return "enabled", nil
+	case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
+		return "LoadState=loaded\nActiveState=active\nSubState=running", nil
+	}
+	return "", nil
+}
+
+func (r *aRuntime) did(word string) int {
+	n := 0
+	for _, c := range r.commands {
+		if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
+			n++
+		}
+	}
+	return n
+}
+
+func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
+	t.Helper()
+	restore := apply.ForTests(t.TempDir(), t.TempDir())
+	t.Cleanup(restore)
+	dir := t.TempDir()
+	was := runtimeConfig
+	runtimeConfig = filepath.Join(dir, "daemon.json")
+	t.Cleanup(func() { runtimeConfig = was })
+	if existing != "" {
+		if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	t.Setenv(internalSuffixVar, "")
+	sys, err := system.For("arch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
+		State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
+	return o, &aRuntime{path: runtimeConfig}, sys
+}
+
+func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
+	t.Helper()
+	raw, err := os.ReadFile(runtimeConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var o map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &o); err != nil {
+		t.Fatal(err)
+	}
+	var list []string
+	_ = json.Unmarshal(o["insecure-registries"], &list)
+	return fmt.Sprint(list), o
+}
+
+func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
+	o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)
+
+	names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The address given and the mesh name the builder pushes by, both on the node's port.
+	if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
+		t.Errorf("trusted by %v", names)
+	}
+	list, object := insecureIn(t)
+	if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
+		t.Errorf("the runtime's list is %s", list)
+	}
+	if string(object["data-root"]) != `"/srv/docker"` {
+		t.Errorf("the machine's own key was not kept: %s", object["data-root"])
+	}
+	if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
+		t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
+	}
+
+	// Re-run: nothing changes, nothing is reloaded.
+	runtime.commands = nil
+	if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+	if runtime.did("reload") != 0 {
+		t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
+	}
+}
+
+// The port is the node's setting for the registry, not the module's default, whatever the operator
+// typed the address with — and loopback, which the runtime trusts already, is not written.
+func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
+	o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
+	t.Setenv(internalSuffixVar, "")
+	if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
+		t.Errorf("names %s", got)
+	}
+	o.Ports = FoundationPorts{}
+	o.Registry = "192.0.2.10:5000"
+	t.Setenv(internalSuffixVar, ".mesh.example")
+	if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
+		t.Errorf("names %s", got)
+	}
+}
+
+// A runtime that still refuses after the write is refused here, with the names, before any push.
+func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
+	o, runtime, sys := trustSetup(t, "")
+	runtime.ignoresReload = true
+	_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
+	if err == nil {
+		t.Fatal("a runtime that does not trust the registry was let through to the push")
+	}
+	if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
+		!strings.Contains(err.Error(), "Nothing has been pushed") {
+		t.Errorf("the refusal does not say what: %v", err)
+	}
+}
+
+// The handover: the `docker` module, applied later, writes the reach the controller composes for
+// the artifact store — the mesh name and the node's port. It finds it there: the file does not
+// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
+// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
+// owns the trust it writes and taking the module away takes that member out.
+func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
+	o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
+	if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+	before, _ := os.ReadFile(runtimeConfig)
+
+	module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
+	  {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
+	   "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
+	  {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
+	   "reload-on":["docker.daemon"]}]}`, runtimeConfig)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	runtime.commands = nil
+	_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
+		runtime.run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	after, _ := os.ReadFile(runtimeConfig)
+	if string(before) != string(after) {
+		t.Errorf("the handover changed the file:\n%s\n%s", before, after)
+	}
+	if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
+		t.Errorf("the handover touched the runtime: %v", runtime.commands)
+	}
+	if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
+		t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
+	}
+}
+
+func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
+	missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
+		[]string{"10.42.0.1:5000", "anchor.internal:5000"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprint(missing) != "[anchor.internal:5000]" {
+		t.Errorf("missing %v", missing)
+	}
+}