diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a1d5382..22cc702 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -292,12 +292,14 @@ type Result struct { Agent string `json:"host-agent,omitempty"` // Registry is the mesh's own artifact store, once it answers. - Registry string `json:"registry,omitempty"` - RegistryReplied int `json:"registry-replied,omitempty"` - RegistryKnown bool `json:"registry-already-registered,omitempty"` - RegistryRunning string `json:"registry-container,omitempty"` - PublishedAs string `json:"control-plane-image,omitempty"` - PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + // RegistryTrusted are the names the runtime was told to trust the registry by, before the push. + RegistryTrusted []string `json:"registry-trusted,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` // Permanent is the control plane as an ordinary module. Permanent string `json:"permanent-container,omitempty"` @@ -555,7 +557,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // Named rather than attempted. Everything from step 6 on is a conversation with a // control plane that a dry run has not raised, so there is nothing to ask and nothing // honest to report about the answers. - say(" would then enrol " + o.Node + ", install the registry from " + + say(" would then enrol " + o.Node + ", have the runtime trust the registry as " + + strings.Join(RegistryNames(o), ", ") + ",") + say(" install the registry from " + o.Catalogue + ", push the control plane's image into it,") say(" reinstall the control plane as a module, and drop " + rewritten.TempName) @@ -656,6 +660,13 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- 7. registry ---------------------------------------------------------------------- say("registry — somewhere for this mesh to keep its own images") + // Trusted by the runtime before anything is pushed to it, by every name the mesh will use for + // it here: the module that writes this trust is applied long after the push (novox/hq issue 92). + trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say) + result.RegistryTrusted = trusted + if err != nil { + return result, failed(StepRegistry, err) + } registry, err := InstallRegistry(ctx, o, d, temporary, say) result.Registry, result.RegistryRunning = registry.Address, registry.Container result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered diff --git a/internal/bootstrap/trust.go b/internal/bootstrap/trust.go new file mode 100644 index 0000000..042aaba --- /dev/null +++ b/internal/bootstrap/trust.go @@ -0,0 +1,232 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92). +// +// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private +// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry +// it has not been told to trust, and the module that tells it — `docker`, which writes the +// artifact store's reach into the runtime's configuration — is applied long after genesis pushes +// the control plane's image. So genesis writes that trust itself, first, with the same writer the +// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and +// never restarted. It then asks the runtime whether it will speak to every name, and refuses with +// the runtime's answer before anything is pushed. +// +// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation +// could hold it, because the retire step re-applies the bundle and takes away a carried record the +// bundle does not name. The `docker` module, applied later, declares the artifact store's reach — +// the mesh name on the node's port — finds it already in the file it has no record of, and takes it +// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). So nothing +// changes on the machine at the handover, no reload; from then on the module owns that member. The +// address the operator gave, when it is not the mesh name, stays as part of the machine's own +// configuration, as the hand-written trust it replaces would have. + +// runtimeConfig is the container runtime's configuration, which the trust is written into. A +// variable so a test writes into a file of its own. +var runtimeConfig = "/etc/docker/daemon.json" + +// runtimeUnit is the runtime's service, reloaded when the file changed. +const runtimeUnit = "docker.service" + +// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable +// and default the controller composes `.` from (mesh-controller internal/overlay), +// which is the name the artifact store's reach — and so every push by the mesh's builder — uses. +const ( + internalSuffixVar = "MESH_INTERNAL_SUFFIX" + defaultInternalSuffix = "internal" +) + +// RegistryNames are the names the mesh will use for its registry on this node, each as host:port. +// +// The address the operator gave, and the node's mesh name: the controller's reach for the artifact +// store, which the builder pushes to without being told. Both on the port the registry module +// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A +// loopback name is left out: the runtime trusts loopback without being told, and the mesh never +// names it to another node. +func RegistryNames(o Options) []string { + port := strconv.Itoa(o.Ports.orDefaults().Registry) + var names []string + add := func(hostport string) { + host, _, err := net.SplitHostPort(hostport) + if err != nil || host == "" || isLoopback(host) { + return + } + for _, n := range names { + if n == hostport { + return + } + } + names = append(names, hostport) + } + if host, _, err := net.SplitHostPort(o.Registry); err == nil { + add(net.JoinHostPort(strings.Trim(host, "[]"), port)) + } + if node := strings.TrimSpace(o.Node); node != "" { + add(net.JoinHostPort(node+"."+internalSuffix(), port)) + } + return names +} + +func internalSuffix() string { + if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" { + return strings.TrimPrefix(v, ".") + } + return defaultInternalSuffix +} + +func isLoopback(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(strings.Trim(host, "[]")) + return ip != nil && ip.IsLoopback() +} + +// trustDeclaration is the declaration genesis applies to write the trust: the same two resources +// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it. +func trustDeclaration(names []string) (*declaration.Declaration, error) { + keys, err := json.Marshal(map[string][]string{"insecure-registries": names}) + if err != nil { + return nil, err + } + raw, err := json.Marshal(map[string]any{ + "declaration": 1, + "resources": []map[string]any{ + {"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644", + "into": "json", "content": string(keys) + "\n"}, + {"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit, + "state": "running", "reload-on": []string{"genesis.registry-trust"}}, + }, + }) + if err != nil { + return nil, err + } + return declaration.ParseTrusted(raw) +} + +// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here, +// then refuses unless the runtime says it will speak plain HTTP to each. +func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner, + say func(string)) ([]string, error) { + + names := RegistryNames(o) + if len(names) == 0 { + say(" trusted only loopback names this registry, which the runtime trusts already") + return nil, nil + } + d, err := trustDeclaration(names) + if err != nil { + return names, err + } + + // One apply at a time on this machine, as ApplyBundle does: a host already running here may be + // writing the same file. + unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") }) + if err != nil { + return names, err + } + report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + unlock() + if err != nil { + return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+ + "The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+ + "runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err) + } + if report.Changed() { + say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig + + ", the runtime reloaded") + } else { + say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so") + } + return names, runtimeWillSpeakTo(ctx, o, run, names) +} + +// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out. +// +// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on +// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error. +func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error { + deadline := time.Now().Add(o.Wait) + var missing []string + var last error + for { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}") + cancel() + last = err + if err == nil { + missing, last = untrusted(out, names) + if last == nil && len(missing) == 0 { + return nil + } + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + if last != nil { + return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last) + } + return fmt.Errorf( + "the container runtime does not trust %s after %s says it should, so it would refuse to push to "+ + "it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+ + "with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+ + "Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit) +} + +// untrusted reads the runtime's registry configuration and gives back the names it would speak +// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in +// a range the runtime trusts. +func untrusted(registryConfig string, names []string) ([]string, error) { + var c struct { + InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"` + IndexConfigs map[string]struct { + Secure bool `json:"Secure"` + } `json:"IndexConfigs"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil { + return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err) + } + var missing []string + for _, name := range names { + if index, ok := c.IndexConfigs[name]; ok && !index.Secure { + continue + } + host, _, _ := net.SplitHostPort(name) + if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) { + continue + } + missing = append(missing, name) + } + return missing, nil +} + +func inAny(ip net.IP, cidrs []string) bool { + for _, cidr := range cidrs { + if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) { + return true + } + } + return false +} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go new file mode 100644 index 0000000..2c6ea09 --- /dev/null +++ b/internal/bootstrap/trust_test.go @@ -0,0 +1,220 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had +// told the container runtime to trust that registry, so the push failed with "server gave HTTP +// response to HTTPS client" — first by the address given, then by the mesh name the builder used, +// and the trust the mesh wrote later named the registry's default port rather than the node's. + +// aRuntime is a container runtime on systemd whose registry configuration is what its file said at +// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry. +type aRuntime struct { + path string + trusted []string + commands []string + // ignoresReload is a runtime started with options of its own, which a reload does not change. + ignoresReload bool +} + +func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + r.commands = append(r.commands, line) + switch { + case name == "docker" && len(args) > 0 && args[0] == "info": + index := map[string]map[string]any{"docker.io": {"Secure": true}} + for _, n := range r.trusted { + index[n] = map[string]any{"Name": n, "Secure": false} + } + out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index}) + return string(out), nil + case name == "systemctl" && strings.Contains(line, "reload"): + if !r.ignoresReload { + r.trusted = r.trusted[:0] + raw, _ := os.ReadFile(r.path) + var c struct { + Insecure []string `json:"insecure-registries"` + } + _ = json.Unmarshal(raw, &c) + r.trusted = append(r.trusted, c.Insecure...) + } + case strings.Contains(line, "is-enabled"): + return "enabled", nil + case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): + return "LoadState=loaded\nActiveState=active\nSubState=running", nil + } + return "", nil +} + +func (r *aRuntime) did(word string) int { + n := 0 + for _, c := range r.commands { + if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) { + n++ + } + } + return n +} + +func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) { + t.Helper() + restore := apply.ForTests(t.TempDir(), t.TempDir()) + t.Cleanup(restore) + dir := t.TempDir() + was := runtimeConfig + runtimeConfig = filepath.Join(dir, "daemon.json") + t.Cleanup(func() { runtimeConfig = was }) + if existing != "" { + if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + } + t.Setenv(internalSuffixVar, "") + sys, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100}, + State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0} + return o, &aRuntime{path: runtimeConfig}, sys +} + +func insecureIn(t *testing.T) (string, map[string]json.RawMessage) { + t.Helper() + raw, err := os.ReadFile(runtimeConfig) + if err != nil { + t.Fatal(err) + } + var o map[string]json.RawMessage + if err := json.Unmarshal(raw, &o); err != nil { + t.Fatal(err) + } + var list []string + _ = json.Unmarshal(o["insecure-registries"], &list) + return fmt.Sprint(list), o +} + +func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) { + o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`) + + names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) + if err != nil { + t.Fatal(err) + } + // The address given and the mesh name the builder pushes by, both on the node's port. + if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" { + t.Errorf("trusted by %v", names) + } + list, object := insecureIn(t) + if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" { + t.Errorf("the runtime's list is %s", list) + } + if string(object["data-root"]) != `"/srv/docker"` { + t.Errorf("the machine's own key was not kept: %s", object["data-root"]) + } + if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 { + t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands) + } + + // Re-run: nothing changes, nothing is reloaded. + runtime.commands = nil + if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { + t.Fatal(err) + } + if runtime.did("reload") != 0 { + t.Errorf("a re-run reloaded the runtime: %v", runtime.commands) + } +} + +// The port is the node's setting for the registry, not the module's default, whatever the operator +// typed the address with — and loopback, which the runtime trusts already, is not written. +func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) { + o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}} + t.Setenv(internalSuffixVar, "") + if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" { + t.Errorf("names %s", got) + } + o.Ports = FoundationPorts{} + o.Registry = "192.0.2.10:5000" + t.Setenv(internalSuffixVar, ".mesh.example") + if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" { + t.Errorf("names %s", got) + } +} + +// A runtime that still refuses after the write is refused here, with the names, before any push. +func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) { + o, runtime, sys := trustSetup(t, "") + runtime.ignoresReload = true + _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) + if err == nil { + t.Fatal("a runtime that does not trust the registry was let through to the push") + } + if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") || + !strings.Contains(err.Error(), "Nothing has been pushed") { + t.Errorf("the refusal does not say what: %v", err) + } +} + +// The handover: the `docker` module, applied later, writes the reach the controller composes for +// the artifact store — the mesh name and the node's port. It finds it there: the file does not +// change, the runtime is not reloaded, and the member it declares is recorded as the module's from +// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module +// owns the trust it writes and taking the module away takes that member out. +func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) { + o, runtime, sys := trustSetup(t, `{"live-restore":true}`) + if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(runtimeConfig) + + module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json", + "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"}, + {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled", + "reload-on":["docker.daemon"]}]}`, runtimeConfig))) + if err != nil { + t.Fatal(err) + } + runtime.commands = nil + _, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared, + runtime.run, nil, nil) + if err != nil { + t.Fatal(err) + } + after, _ := os.ReadFile(runtimeConfig) + if string(before) != string(after) { + t.Errorf("the handover changed the file:\n%s\n%s", before, after) + } + if runtime.did("reload") != 0 || runtime.did("restart") != 0 { + t.Errorf("the handover touched the runtime: %v", runtime.commands) + } + if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 { + t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into) + } +} + +func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { + missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`, + []string{"10.42.0.1:5000", "anchor.internal:5000"}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(missing) != "[anchor.internal:5000]" { + t.Errorf("missing %v", missing) + } +}