mesh-host/internal/bootstrap/trust.go

package bootstrap

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/apply"
	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92).
//
// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private
// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry
// it has not been told to trust, and the module that tells it — `docker`, which writes the
// artifact store's reach into the runtime's configuration — is applied long after genesis pushes
// the control plane's image. So genesis writes that trust itself, first, with the same writer the
// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and
// never restarted. It then asks the runtime whether it will speak to every name, and refuses with
// the runtime's answer before anything is pushed.
//
// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation
// could hold it, because the retire step re-applies the bundle and takes away a carried record the
// bundle does not name. The `docker` module, applied later, declares the artifact store's reach —
// the mesh name on the node's port — finds it already in the file it has no record of, and takes it
// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member
// does not change at the handover. The module's first apply usually still changes the file and
// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`,
// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name,
// stays as part of the machine's own configuration, as the hand-written trust it replaces would have.
//
// **This rests on an assumption about order.** The module takes the mesh name's member over only
// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the
// network step (16), and once the node is on the network the controller composes the reach as
// `<node>.<suffix>:<port>`.
// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:<port>`, as it is before the
// network — its record would exist without the mesh name, and the member would stay the machine's
// for good: undeclaring the module would never take it out. Installing the node's seat holders
// earlier in genesis must revisit this.
//
// **The suffix** is read from this process's environment, and the controller reads its own: a mesh
// with a custom `MESH_INTERNAL_SUFFIX` sets it for both.

// runtimeConfig is the container runtime's configuration, which the trust is written into. A
// variable so a test writes into a file of its own.
var runtimeConfig = "/etc/docker/daemon.json"

// runtimeUnit is the runtime's service, reloaded when the file changed.
const runtimeUnit = "docker.service"

// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable
// and default the controller composes `<node>.<suffix>` from (mesh-controller internal/overlay),
// which is the name the artifact store's reach — and so every push by the mesh's builder — uses.
const (
	internalSuffixVar     = "MESH_INTERNAL_SUFFIX"
	defaultInternalSuffix = "internal"
)

// RegistryNames are the names the mesh will use for its registry on this node, each as host:port.
//
// The address the operator gave, and the node's mesh name: the controller's reach for the artifact
// store, which the builder pushes to without being told. Both on the port the registry module
// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A
// loopback name is left out: the runtime trusts loopback without being told, and the mesh never
// names it to another node.
func RegistryNames(o Options) []string {
	port := strconv.Itoa(o.Ports.orDefaults().Registry)
	var names []string
	add := func(hostport string) {
		host, _, err := net.SplitHostPort(hostport)
		if err != nil || host == "" || isLoopback(host) {
			return
		}
		for _, n := range names {
			if n == hostport {
				return
			}
		}
		names = append(names, hostport)
	}
	if host, _, err := net.SplitHostPort(o.Registry); err == nil {
		add(net.JoinHostPort(strings.Trim(host, "[]"), port))
	}
	if node := strings.TrimSpace(o.Node); node != "" {
		add(net.JoinHostPort(node+"."+internalSuffix(), port))
	}
	return names
}

func internalSuffix() string {
	if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" {
		return strings.TrimPrefix(v, ".")
	}
	return defaultInternalSuffix
}

func isLoopback(host string) bool {
	if strings.EqualFold(host, "localhost") {
		return true
	}
	ip := net.ParseIP(strings.Trim(host, "[]"))
	return ip != nil && ip.IsLoopback()
}

// trustDeclaration is the declaration genesis applies to write the trust: the same two resources
// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it.
func trustDeclaration(names []string) (*declaration.Declaration, error) {
	keys, err := json.Marshal(map[string][]string{"insecure-registries": names})
	if err != nil {
		return nil, err
	}
	raw, err := json.Marshal(map[string]any{
		"declaration": 1,
		"resources": []map[string]any{
			{"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644",
				"into": "json", "content": string(keys) + "\n"},
			{"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit,
				"state": "running", "reload-on": []string{"genesis.registry-trust"}},
		},
	})
	if err != nil {
		return nil, err
	}
	return declaration.ParseTrusted(raw)
}

// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here,
// then refuses unless the runtime says it will speak plain HTTP to each.
func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner,
	say func(string)) ([]string, error) {

	names := RegistryNames(o)
	if len(names) == 0 {
		say("  trusted           only loopback names this registry, which the runtime trusts already")
		return nil, nil
	}
	d, err := trustDeclaration(names)
	if err != nil {
		return names, err
	}

	// One apply at a time on this machine, as ApplyBundle does: a host already running here may be
	// writing the same file.
	unlock, err := store.Lock(o.State, func() { say("  waiting           another apply holds this node's state") })
	if err != nil {
		return names, err
	}
	report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run,
		func(line string) { say("  " + strings.TrimPrefix(line, "  ")) }, refuseSealed)
	unlock()
	if err != nil {
		return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+
			"The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+
			"runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err)
	}
	if report.Changed() {
		say("  trusted           " + strings.Join(names, ", ") + " — written into " + runtimeConfig +
			", the runtime reloaded")
	} else {
		say("  trusted           " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so")
	}
	return names, runtimeWillSpeakTo(ctx, o, run, names)
}

// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out.
//
// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on
// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error.
func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error {
	deadline := time.Now().Add(o.Wait)
	var missing []string
	var last error
	for {
		asking, cancel := context.WithTimeout(ctx, o.Timeout)
		out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}")
		cancel()
		last = err
		if err == nil {
			missing, last = untrusted(out, names)
			if last == nil && len(missing) == 0 {
				return nil
			}
		}
		if time.Now().After(deadline) {
			break
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(answerEvery):
		}
	}
	if last != nil {
		return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last)
	}
	return fmt.Errorf(
		"the container runtime does not trust %s after %s says it should, so it would refuse to push to "+
			"it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+
			"with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+
			"Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit)
}

// untrusted reads the runtime's registry configuration and gives back the names it would speak
// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in
// a range the runtime trusts.
func untrusted(registryConfig string, names []string) ([]string, error) {
	var c struct {
		InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"`
		IndexConfigs          map[string]struct {
			Secure bool `json:"Secure"`
		} `json:"IndexConfigs"`
	}
	if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil {
		return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err)
	}
	var missing []string
	for _, name := range names {
		if index, ok := c.IndexConfigs[name]; ok && !index.Secure {
			continue
		}
		host, _, _ := net.SplitHostPort(name)
		if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) {
			continue
		}
		missing = append(missing, name)
	}
	return missing, nil
}

func inAny(ip net.IP, cidrs []string) bool {
	for _, cidr := range cidrs {
		if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) {
			return true
		}
	}
	return false
}

// procRoot is where a process's command line is read from. A variable so a test gives its own.
var procRoot = "/proc"

// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known
// about it without writing anything, asked at step 1 so a machine that cannot take the trust is
// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92).
//
//   - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to
//     write over anything else (ADR 0102).
//   - The runtime's service is known to systemd and active: the trust is taken by reloading it.
//   - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both
//     as a flag and in its file, so it would refuse the reload and keep distrusting the registry.
func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error {
	names := RegistryNames(o)
	if len(names) == 0 {
		return nil
	}
	if raw, err := os.ReadFile(runtimeConfig); err == nil {
		var object map[string]json.RawMessage
		if len(strings.TrimSpace(string(raw))) > 0 {
			if err := json.Unmarshal(raw, &object); err != nil || object == nil {
				return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+
					"(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+
					"or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", "))
			}
		}
	} else if !os.IsNotExist(err) {
		return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err)
	}

	asking, cancel := context.WithTimeout(ctx, o.Timeout)
	out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID")
	cancel()
	if err != nil {
		return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+
			"reloading it; nothing has been changed", runtimeUnit, err)
	}
	props := map[string]string{}
	for _, line := range strings.Split(out, "\n") {
		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
			props[k] = v
		}
	}
	if props["LoadState"] != "loaded" || props["ActiveState"] != "active" {
		return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+
			"Genesis needs the runtime run by systemd as %s; nothing has been changed",
			runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]),
			strings.Join(names, ", "), runtimeUnit)
	}
	if pid := props["MainPID"]; pid != "" && pid != "0" {
		if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil {
			for _, arg := range strings.Split(string(cmdline), "\x00") {
				if strings.HasPrefix(arg, "--insecure-registry") {
					return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+
						"given both as a flag and in %s — so it would refuse the reload that makes it trust "+
						"%s. Move the flag's registries into the file and restart the runtime once; nothing "+
						"has been changed", arg, runtimeConfig, strings.Join(names, ", "))
				}
			}
		}
	}
	say("  registry trust    " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded")
	return nil
}

func orUnknown(s string) string {
	if s == "" {
		return "unknown"
	}
	return s
}