mesh-host/internal/bootstrap/trust_test.go

package bootstrap

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/declaration"
	"github.com/novox/mesh-host/internal/apply"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
// told the container runtime to trust that registry, so the push failed with "server gave HTTP
// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
// and the trust the mesh wrote later named the registry's default port rather than the node's.

// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
type aRuntime struct {
	path     string
	trusted  []string
	commands []string
	// ignoresReload is a runtime started with options of its own, which a reload does not change.
	ignoresReload bool
}

func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
	line := name + " " + strings.Join(args, " ")
	r.commands = append(r.commands, line)
	switch {
	case name == "docker" && len(args) > 0 && args[0] == "info":
		index := map[string]map[string]any{"docker.io": {"Secure": true}}
		for _, n := range r.trusted {
			index[n] = map[string]any{"Name": n, "Secure": false}
		}
		out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
		return string(out), nil
	case name == "systemctl" && strings.Contains(line, "reload"):
		if !r.ignoresReload {
			r.trusted = r.trusted[:0]
			raw, _ := os.ReadFile(r.path)
			var c struct {
				Insecure []string `json:"insecure-registries"`
			}
			_ = json.Unmarshal(raw, &c)
			r.trusted = append(r.trusted, c.Insecure...)
		}
	case strings.Contains(line, "is-enabled"):
		return "enabled", nil
	case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
		return "LoadState=loaded\nActiveState=active\nSubState=running", nil
	}
	return "", nil
}

func (r *aRuntime) did(word string) int {
	n := 0
	for _, c := range r.commands {
		if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
			n++
		}
	}
	return n
}

func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
	t.Helper()
	restore := apply.ForTests(t.TempDir(), t.TempDir())
	t.Cleanup(restore)
	dir := t.TempDir()
	was := runtimeConfig
	runtimeConfig = filepath.Join(dir, "daemon.json")
	t.Cleanup(func() { runtimeConfig = was })
	if existing != "" {
		if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	t.Setenv(internalSuffixVar, "")
	sys, err := system.For("arch")
	if err != nil {
		t.Fatal(err)
	}
	o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
		State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
	return o, &aRuntime{path: runtimeConfig}, sys
}

func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
	t.Helper()
	raw, err := os.ReadFile(runtimeConfig)
	if err != nil {
		t.Fatal(err)
	}
	var o map[string]json.RawMessage
	if err := json.Unmarshal(raw, &o); err != nil {
		t.Fatal(err)
	}
	var list []string
	_ = json.Unmarshal(o["insecure-registries"], &list)
	return fmt.Sprint(list), o
}

func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
	o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)

	names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	// The address given and the mesh name the builder pushes by, both on the node's port.
	if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
		t.Errorf("trusted by %v", names)
	}
	list, object := insecureIn(t)
	if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
		t.Errorf("the runtime's list is %s", list)
	}
	if string(object["data-root"]) != `"/srv/docker"` {
		t.Errorf("the machine's own key was not kept: %s", object["data-root"])
	}
	if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
		t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
	}

	// Re-run: nothing changes, nothing is reloaded.
	runtime.commands = nil
	if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
		t.Fatal(err)
	}
	if runtime.did("reload") != 0 {
		t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
	}
}

// The port is the node's setting for the registry, not the module's default, whatever the operator
// typed the address with — and loopback, which the runtime trusts already, is not written.
func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
	o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
	t.Setenv(internalSuffixVar, "")
	if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
		t.Errorf("names %s", got)
	}
	o.Ports = FoundationPorts{}
	o.Registry = "192.0.2.10:5000"
	t.Setenv(internalSuffixVar, ".mesh.example")
	if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
		t.Errorf("names %s", got)
	}
}

// A runtime that still refuses after the write is refused here, with the names, before any push.
func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
	o, runtime, sys := trustSetup(t, "")
	runtime.ignoresReload = true
	_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
	if err == nil {
		t.Fatal("a runtime that does not trust the registry was let through to the push")
	}
	if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
		!strings.Contains(err.Error(), "Nothing has been pushed") {
		t.Errorf("the refusal does not say what: %v", err)
	}
}

// The handover: the `docker` module, applied later, writes the reach the controller composes for
// the artifact store — the mesh name and the node's port. It finds it there: the file does not
// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
// owns the trust it writes and taking the module away takes that member out.
func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
	o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
	if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
		t.Fatal(err)
	}
	before, _ := os.ReadFile(runtimeConfig)

	module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
	   "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
	  {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
	   "reload-on":["docker.daemon"]}]}`, runtimeConfig)))
	if err != nil {
		t.Fatal(err)
	}
	runtime.commands = nil
	_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
		runtime.run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	after, _ := os.ReadFile(runtimeConfig)
	if string(before) != string(after) {
		t.Errorf("the handover changed the file:\n%s\n%s", before, after)
	}
	if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
		t.Errorf("the handover touched the runtime: %v", runtime.commands)
	}
	if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
		t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
	}
}

func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
	missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
		[]string{"10.42.0.1:5000", "anchor.internal:5000"})
	if err != nil {
		t.Fatal(err)
	}
	if fmt.Sprint(missing) != "[anchor.internal:5000]" {
		t.Errorf("missing %v", missing)
	}
}

// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves
// as dockerd does: a push to a registry it has not been told to trust is refused with the error the
// first genesis met. The trust must be written and confirmed before anything is tagged or pushed;
// moving the trust after the push, or dropping it, fails this test.
func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
	o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`)
	// The holder's state directory and its account, as distribution's manifest declares them (ENG-5).
	stateDir := filepath.Join(t.TempDir(), "registry")
	o.Catalogue = catalogueWith(t, RegistryModule, strings.Replace(upstreamRegistryManifest,
		`{"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}`,
		`{"id": "account", "type": "user", "name": "distribution"},
    {"id": "state", "type": "directory", "path": "`+stateDir+`", "mode": "0700", "owner": "distribution"}`, 1))
	o.Registry = "192.0.2.10:5100"
	asked, was := []string{}, lookupAccount
	lookupAccount = func(name string) (int, int, error) {
		asked = append(asked, name)
		return os.Getuid(), os.Getgid(), nil
	}
	t.Cleanup(func() { lookupAccount = was })
	o.PushCredentialFile = filepath.Join(stateDir, "bootstrap-push")
	// Before the node-engine made distribution's state directory, nothing is written: the program could not read it.
	if _, err := WritePushCredential(o.Catalogue, o.PushCredentialFile); err == nil {
		t.Errorf("the push credential was written before its holder's state directory existed")
	} else if _, err := os.Stat(stateDir); err == nil {
		t.Errorf("the bootstrap made the holder's state directory itself")
	}
	if err := os.Mkdir(stateDir, 0o700); err != nil {
		t.Fatal(err)
	}

	pushed := false
	mesh := aMeshThatAgrees(nil)
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "systemctl" || (name == "docker" && args[0] == "info") {
			return runtime.run(ctx, name, args...)
		}
		runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " "))
		if name == "docker" {
			switch args[0] {
			case "tag":
				return "", nil
			case "push":
				trusted := false
				for _, n := range runtime.trusted {
					trusted = trusted || strings.HasPrefix(args[1], n+"/")
				}
				if !trusted {
					return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry)
				}
				pushed = true
				return "", nil
			case "image":
				return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
			}
		}
		return mesh(name, args)
	}
	d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) {
		if strings.HasSuffix(url, "/v2/") || pushed {
			return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
		}
		return http.StatusNotFound, "", nil
	}}
	var result Result
	// And the one-off credential for the holders' first images (ENG-5): written once, mode 0600, one line of 64 hex digits,
	// named in the result and never put in it.
	published, err := RegistryAndPublish(context.Background(), o, d, sys,
		controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {})
	if err != nil {
		t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands)
	}
	if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") {
		t.Errorf("published as %q", published.Reference)
	}

	firstPush, reloaded, confirmed := -1, -1, -1
	for i, c := range runtime.commands {
		switch {
		case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")):
			firstPush = i
		case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"):
			reloaded = i
		case strings.HasPrefix(c, "docker info"):
			confirmed = i
		}
	}
	if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) {
		t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v",
			reloaded, confirmed, firstPush, runtime.commands)
	}
	if raw, err := os.ReadFile(o.PushCredentialFile); err != nil || !regexp.MustCompile(`^[0-9a-f]{64}\n$`).Match(raw) {
		t.Errorf("the push credential is %q (%v)", raw, err)
	} else if st, _ := os.Stat(o.PushCredentialFile); st.Mode().Perm() != 0o600 ||
		int(st.Sys().(*syscall.Stat_t).Uid) != os.Getuid() || int(st.Sys().(*syscall.Stat_t).Gid) != os.Getgid() {
		t.Errorf("the push credential is mode %v, owner %d:%d", st.Mode().Perm(), st.Sys().(*syscall.Stat_t).Uid, st.Sys().(*syscall.Stat_t).Gid)
	}
	if fmt.Sprint(asked) != "[distribution distribution]" {
		t.Errorf("the push credential's owner was asked as %v, not the account distribution's state directory names", asked)
	}
	if result.PushCredentialFile != o.PushCredentialFile {
		t.Errorf("the result names %q", result.PushCredentialFile)
	}
	if again, err := MintPushCredential(o.PushCredentialFile, os.Getuid(), os.Getgid()); err != nil || again.Token() == "" {
		t.Errorf("a credential of an earlier run was not replaced: %v", err)
	} else if again.Forget(); again.Token() != "" {
		t.Errorf("a forgotten credential still holds its token")
	}
	if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" {
		t.Errorf("the result says it trusted %v", result.RegistryTrusted)
	}
}

// Preflight asks, without writing, what would stop the trust from taking at step 7.
func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) {
	proc := t.TempDir()
	was := procRoot
	procRoot = proc
	t.Cleanup(func() { procRoot = was })
	if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil {
		t.Fatal(err)
	}
	unit := func(load, active string) Runner {
		return func(_ context.Context, name string, args ...string) (string, error) {
			return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil
		}
	}
	cmdline := func(args ...string) {
		if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	cmdline("/usr/bin/dockerd", "-H", "fd://")

	for _, c := range []struct {
		name, file string
		run        Runner
		flag       bool
		refused    string
	}{
		{"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""},
		{"no file yet", "", unit("loaded", "active"), false, ""},
		{"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"},
		{"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"},
		{"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"},
	} {
		t.Run(c.name, func(t *testing.T) {
			o, _, _ := trustSetup(t, c.file)
			if c.flag {
				cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000")
				defer cmdline("/usr/bin/dockerd", "-H", "fd://")
			}
			before, _ := os.ReadFile(runtimeConfig)
			err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {})
			switch {
			case c.refused == "" && err != nil:
				t.Errorf("refused: %v", err)
			case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)):
				t.Errorf("wanted a refusal naming %q, got %v", c.refused, err)
			}
			if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) {
				t.Error("preflight wrote the runtime's file")
			}
		})
	}
}