From 41e640d66888072454a0b17eb01b8bc6aa182690 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:13:11 +0200 Subject: [PATCH 1/2] Have genesis trust its registry before pushing to it (issue 92) Genesis pushed the control plane's image to the mesh's plain-HTTP registry before anything told the container runtime to trust it; the docker module that writes that trust is applied long after. Genesis now writes the trust itself at the registry step, with the same into-json writer and a reload, for the address given and the node's mesh name on the node's registry port, and refuses before any push unless the runtime says it trusts each. The docker module later finds the mesh name there and takes it over without a change. --- internal/bootstrap/bootstrap.go | 25 +++- internal/bootstrap/trust.go | 232 +++++++++++++++++++++++++++++++ internal/bootstrap/trust_test.go | 220 +++++++++++++++++++++++++++++ 3 files changed, 470 insertions(+), 7 deletions(-) create mode 100644 internal/bootstrap/trust.go create mode 100644 internal/bootstrap/trust_test.go diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a1d5382..22cc702 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -292,12 +292,14 @@ type Result struct { Agent string `json:"host-agent,omitempty"` // Registry is the mesh's own artifact store, once it answers. - Registry string `json:"registry,omitempty"` - RegistryReplied int `json:"registry-replied,omitempty"` - RegistryKnown bool `json:"registry-already-registered,omitempty"` - RegistryRunning string `json:"registry-container,omitempty"` - PublishedAs string `json:"control-plane-image,omitempty"` - PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + // RegistryTrusted are the names the runtime was told to trust the registry by, before the push. + RegistryTrusted []string `json:"registry-trusted,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` // Permanent is the control plane as an ordinary module. Permanent string `json:"permanent-container,omitempty"` @@ -555,7 +557,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // Named rather than attempted. Everything from step 6 on is a conversation with a // control plane that a dry run has not raised, so there is nothing to ask and nothing // honest to report about the answers. - say(" would then enrol " + o.Node + ", install the registry from " + + say(" would then enrol " + o.Node + ", have the runtime trust the registry as " + + strings.Join(RegistryNames(o), ", ") + ",") + say(" install the registry from " + o.Catalogue + ", push the control plane's image into it,") say(" reinstall the control plane as a module, and drop " + rewritten.TempName) @@ -656,6 +660,13 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- 7. registry ---------------------------------------------------------------------- say("registry — somewhere for this mesh to keep its own images") + // Trusted by the runtime before anything is pushed to it, by every name the mesh will use for + // it here: the module that writes this trust is applied long after the push (novox/hq issue 92). + trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say) + result.RegistryTrusted = trusted + if err != nil { + return result, failed(StepRegistry, err) + } registry, err := InstallRegistry(ctx, o, d, temporary, say) result.Registry, result.RegistryRunning = registry.Address, registry.Container result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered diff --git a/internal/bootstrap/trust.go b/internal/bootstrap/trust.go new file mode 100644 index 0000000..042aaba --- /dev/null +++ b/internal/bootstrap/trust.go @@ -0,0 +1,232 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92). +// +// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private +// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry +// it has not been told to trust, and the module that tells it — `docker`, which writes the +// artifact store's reach into the runtime's configuration — is applied long after genesis pushes +// the control plane's image. So genesis writes that trust itself, first, with the same writer the +// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and +// never restarted. It then asks the runtime whether it will speak to every name, and refuses with +// the runtime's answer before anything is pushed. +// +// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation +// could hold it, because the retire step re-applies the bundle and takes away a carried record the +// bundle does not name. The `docker` module, applied later, declares the artifact store's reach — +// the mesh name on the node's port — finds it already in the file it has no record of, and takes it +// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). So nothing +// changes on the machine at the handover, no reload; from then on the module owns that member. The +// address the operator gave, when it is not the mesh name, stays as part of the machine's own +// configuration, as the hand-written trust it replaces would have. + +// runtimeConfig is the container runtime's configuration, which the trust is written into. A +// variable so a test writes into a file of its own. +var runtimeConfig = "/etc/docker/daemon.json" + +// runtimeUnit is the runtime's service, reloaded when the file changed. +const runtimeUnit = "docker.service" + +// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable +// and default the controller composes `.` from (mesh-controller internal/overlay), +// which is the name the artifact store's reach — and so every push by the mesh's builder — uses. +const ( + internalSuffixVar = "MESH_INTERNAL_SUFFIX" + defaultInternalSuffix = "internal" +) + +// RegistryNames are the names the mesh will use for its registry on this node, each as host:port. +// +// The address the operator gave, and the node's mesh name: the controller's reach for the artifact +// store, which the builder pushes to without being told. Both on the port the registry module +// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A +// loopback name is left out: the runtime trusts loopback without being told, and the mesh never +// names it to another node. +func RegistryNames(o Options) []string { + port := strconv.Itoa(o.Ports.orDefaults().Registry) + var names []string + add := func(hostport string) { + host, _, err := net.SplitHostPort(hostport) + if err != nil || host == "" || isLoopback(host) { + return + } + for _, n := range names { + if n == hostport { + return + } + } + names = append(names, hostport) + } + if host, _, err := net.SplitHostPort(o.Registry); err == nil { + add(net.JoinHostPort(strings.Trim(host, "[]"), port)) + } + if node := strings.TrimSpace(o.Node); node != "" { + add(net.JoinHostPort(node+"."+internalSuffix(), port)) + } + return names +} + +func internalSuffix() string { + if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" { + return strings.TrimPrefix(v, ".") + } + return defaultInternalSuffix +} + +func isLoopback(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(strings.Trim(host, "[]")) + return ip != nil && ip.IsLoopback() +} + +// trustDeclaration is the declaration genesis applies to write the trust: the same two resources +// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it. +func trustDeclaration(names []string) (*declaration.Declaration, error) { + keys, err := json.Marshal(map[string][]string{"insecure-registries": names}) + if err != nil { + return nil, err + } + raw, err := json.Marshal(map[string]any{ + "declaration": 1, + "resources": []map[string]any{ + {"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644", + "into": "json", "content": string(keys) + "\n"}, + {"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit, + "state": "running", "reload-on": []string{"genesis.registry-trust"}}, + }, + }) + if err != nil { + return nil, err + } + return declaration.ParseTrusted(raw) +} + +// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here, +// then refuses unless the runtime says it will speak plain HTTP to each. +func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner, + say func(string)) ([]string, error) { + + names := RegistryNames(o) + if len(names) == 0 { + say(" trusted only loopback names this registry, which the runtime trusts already") + return nil, nil + } + d, err := trustDeclaration(names) + if err != nil { + return names, err + } + + // One apply at a time on this machine, as ApplyBundle does: a host already running here may be + // writing the same file. + unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") }) + if err != nil { + return names, err + } + report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + unlock() + if err != nil { + return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+ + "The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+ + "runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err) + } + if report.Changed() { + say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig + + ", the runtime reloaded") + } else { + say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so") + } + return names, runtimeWillSpeakTo(ctx, o, run, names) +} + +// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out. +// +// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on +// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error. +func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error { + deadline := time.Now().Add(o.Wait) + var missing []string + var last error + for { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}") + cancel() + last = err + if err == nil { + missing, last = untrusted(out, names) + if last == nil && len(missing) == 0 { + return nil + } + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + if last != nil { + return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last) + } + return fmt.Errorf( + "the container runtime does not trust %s after %s says it should, so it would refuse to push to "+ + "it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+ + "with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+ + "Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit) +} + +// untrusted reads the runtime's registry configuration and gives back the names it would speak +// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in +// a range the runtime trusts. +func untrusted(registryConfig string, names []string) ([]string, error) { + var c struct { + InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"` + IndexConfigs map[string]struct { + Secure bool `json:"Secure"` + } `json:"IndexConfigs"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil { + return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err) + } + var missing []string + for _, name := range names { + if index, ok := c.IndexConfigs[name]; ok && !index.Secure { + continue + } + host, _, _ := net.SplitHostPort(name) + if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) { + continue + } + missing = append(missing, name) + } + return missing, nil +} + +func inAny(ip net.IP, cidrs []string) bool { + for _, cidr := range cidrs { + if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) { + return true + } + } + return false +} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go new file mode 100644 index 0000000..2c6ea09 --- /dev/null +++ b/internal/bootstrap/trust_test.go @@ -0,0 +1,220 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had +// told the container runtime to trust that registry, so the push failed with "server gave HTTP +// response to HTTPS client" — first by the address given, then by the mesh name the builder used, +// and the trust the mesh wrote later named the registry's default port rather than the node's. + +// aRuntime is a container runtime on systemd whose registry configuration is what its file said at +// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry. +type aRuntime struct { + path string + trusted []string + commands []string + // ignoresReload is a runtime started with options of its own, which a reload does not change. + ignoresReload bool +} + +func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + r.commands = append(r.commands, line) + switch { + case name == "docker" && len(args) > 0 && args[0] == "info": + index := map[string]map[string]any{"docker.io": {"Secure": true}} + for _, n := range r.trusted { + index[n] = map[string]any{"Name": n, "Secure": false} + } + out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index}) + return string(out), nil + case name == "systemctl" && strings.Contains(line, "reload"): + if !r.ignoresReload { + r.trusted = r.trusted[:0] + raw, _ := os.ReadFile(r.path) + var c struct { + Insecure []string `json:"insecure-registries"` + } + _ = json.Unmarshal(raw, &c) + r.trusted = append(r.trusted, c.Insecure...) + } + case strings.Contains(line, "is-enabled"): + return "enabled", nil + case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): + return "LoadState=loaded\nActiveState=active\nSubState=running", nil + } + return "", nil +} + +func (r *aRuntime) did(word string) int { + n := 0 + for _, c := range r.commands { + if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) { + n++ + } + } + return n +} + +func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) { + t.Helper() + restore := apply.ForTests(t.TempDir(), t.TempDir()) + t.Cleanup(restore) + dir := t.TempDir() + was := runtimeConfig + runtimeConfig = filepath.Join(dir, "daemon.json") + t.Cleanup(func() { runtimeConfig = was }) + if existing != "" { + if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + } + t.Setenv(internalSuffixVar, "") + sys, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100}, + State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0} + return o, &aRuntime{path: runtimeConfig}, sys +} + +func insecureIn(t *testing.T) (string, map[string]json.RawMessage) { + t.Helper() + raw, err := os.ReadFile(runtimeConfig) + if err != nil { + t.Fatal(err) + } + var o map[string]json.RawMessage + if err := json.Unmarshal(raw, &o); err != nil { + t.Fatal(err) + } + var list []string + _ = json.Unmarshal(o["insecure-registries"], &list) + return fmt.Sprint(list), o +} + +func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) { + o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`) + + names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) + if err != nil { + t.Fatal(err) + } + // The address given and the mesh name the builder pushes by, both on the node's port. + if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" { + t.Errorf("trusted by %v", names) + } + list, object := insecureIn(t) + if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" { + t.Errorf("the runtime's list is %s", list) + } + if string(object["data-root"]) != `"/srv/docker"` { + t.Errorf("the machine's own key was not kept: %s", object["data-root"]) + } + if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 { + t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands) + } + + // Re-run: nothing changes, nothing is reloaded. + runtime.commands = nil + if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { + t.Fatal(err) + } + if runtime.did("reload") != 0 { + t.Errorf("a re-run reloaded the runtime: %v", runtime.commands) + } +} + +// The port is the node's setting for the registry, not the module's default, whatever the operator +// typed the address with — and loopback, which the runtime trusts already, is not written. +func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) { + o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}} + t.Setenv(internalSuffixVar, "") + if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" { + t.Errorf("names %s", got) + } + o.Ports = FoundationPorts{} + o.Registry = "192.0.2.10:5000" + t.Setenv(internalSuffixVar, ".mesh.example") + if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" { + t.Errorf("names %s", got) + } +} + +// A runtime that still refuses after the write is refused here, with the names, before any push. +func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) { + o, runtime, sys := trustSetup(t, "") + runtime.ignoresReload = true + _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) + if err == nil { + t.Fatal("a runtime that does not trust the registry was let through to the push") + } + if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") || + !strings.Contains(err.Error(), "Nothing has been pushed") { + t.Errorf("the refusal does not say what: %v", err) + } +} + +// The handover: the `docker` module, applied later, writes the reach the controller composes for +// the artifact store — the mesh name and the node's port. It finds it there: the file does not +// change, the runtime is not reloaded, and the member it declares is recorded as the module's from +// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module +// owns the trust it writes and taking the module away takes that member out. +func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) { + o, runtime, sys := trustSetup(t, `{"live-restore":true}`) + if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(runtimeConfig) + + module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json", + "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"}, + {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled", + "reload-on":["docker.daemon"]}]}`, runtimeConfig))) + if err != nil { + t.Fatal(err) + } + runtime.commands = nil + _, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared, + runtime.run, nil, nil) + if err != nil { + t.Fatal(err) + } + after, _ := os.ReadFile(runtimeConfig) + if string(before) != string(after) { + t.Errorf("the handover changed the file:\n%s\n%s", before, after) + } + if runtime.did("reload") != 0 || runtime.did("restart") != 0 { + t.Errorf("the handover touched the runtime: %v", runtime.commands) + } + if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 { + t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into) + } +} + +func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { + missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`, + []string{"10.42.0.1:5000", "anchor.internal:5000"}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(missing) != "[anchor.internal:5000]" { + t.Errorf("missing %v", missing) + } +} From 0818e4c29ae48e5b909f4eec97f1130951448c35 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:29:14 +0200 Subject: [PATCH 2/2] Hold the trust-before-push order by a test, and refuse early what would stop it (issue 92) The fix is an order, and nothing failed if the trust call moved after the push or went: steps 7 and 8 are now one function a test drives against a runtime that refuses an untrusted push. Preflight now refuses, before anything is raised, a runtime file that is no JSON object, a docker.service that is not loaded and active, and a dockerd started with --insecure-registry, which would refuse the reload. The comment names the order the docker module's takeover rests on. --- internal/bootstrap/bootstrap.go | 71 ++++++++++------- internal/bootstrap/preflight.go | 7 ++ internal/bootstrap/trust.go | 93 +++++++++++++++++++++- internal/bootstrap/trust_test.go | 127 +++++++++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 30 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 22cc702..4c4e46a 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -36,6 +36,7 @@ import ( "time" "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/tunnel" ) @@ -658,33 +659,10 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepOperator, err) } - // ---- 7. registry ---------------------------------------------------------------------- - say("registry — somewhere for this mesh to keep its own images") - // Trusted by the runtime before anything is pushed to it, by every name the mesh will use for - // it here: the module that writes this trust is applied long after the push (novox/hq issue 92). - trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say) - result.RegistryTrusted = trusted + // ---- 7. registry, 8. publish ----------------------------------------------------------- + published, err := RegistryAndPublish(ctx, o, d, sys, temporary, controlPlaneImage, &result, say) if err != nil { - return result, failed(StepRegistry, err) - } - registry, err := InstallRegistry(ctx, o, d, temporary, say) - result.Registry, result.RegistryRunning = registry.Address, registry.Container - result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered - if err != nil { - return result, failed(StepRegistry, err) - } - - // ---- 8. publish ----------------------------------------------------------------------- - say("publish — the control plane's image gets its first manifest digest") - // **The image this mesh built, not the one the installer carried.** The carried one is the - // builder; publishing it here would put a builder in the registry under the control plane's - // name and install it as the control plane — which is exactly what happened the first time - // this ran, and presented as a control plane that started, printed a builder's usage, and - // exited cleanly over and over. - published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say) - result.PublishedAs, result.PublishedAlready = published.Reference, published.Already - if err != nil { - return result, failed(StepPublish, err) + return result, err } // ---- 9. control plane ----------------------------------------------------------------- @@ -814,3 +792,44 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, nil } + +// RegistryAndPublish is steps 7 and 8: the runtime trusts the mesh's registry, the registry is +// installed and answers, and the control plane's image is pushed into it. +// +// **One function so the order is held by a test** (novox/hq issue 92). The runtime refuses plain +// HTTP to a registry it has not been told to trust, and the module that tells it is applied long +// after the push — so the trust is written and confirmed here, before anything is tagged or pushed. +// A refusal names the step it happened in. +func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.System, temporary controlPlane, + controlPlaneImage string, result *Result, say func(string)) (Published, error) { + + // ---- 7. registry ---------------------------------------------------------------------- + say("registry — somewhere for this mesh to keep its own images") + // Trusted by the runtime before anything is pushed to it, by every name the mesh will use for + // it here: the module that writes this trust is applied long after the push (novox/hq issue 92). + trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say) + result.RegistryTrusted = trusted + if err != nil { + return Published{}, failed(StepRegistry, err) + } + registry, err := InstallRegistry(ctx, o, d, temporary, say) + result.Registry, result.RegistryRunning = registry.Address, registry.Container + result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered + if err != nil { + return Published{}, failed(StepRegistry, err) + } + + // ---- 8. publish ----------------------------------------------------------------------- + say("publish — the control plane's image gets its first manifest digest") + // **The image this mesh built, not the one the installer carried.** The carried one is the + // builder; publishing it here would put a builder in the registry under the control plane's + // name and install it as the control plane — which is exactly what happened the first time + // this ran, and presented as a control plane that started, printed a builder's usage, and + // exited cleanly over and over. + published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say) + result.PublishedAs, result.PublishedAlready = published.Reference, published.Already + if err != nil { + return published, failed(StepPublish, err) + } + return published, nil +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index 7b8d145..f3502a0 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -110,6 +110,13 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil { return nil, err } + // The registry's trust is written at step 7, before the push (novox/hq issue 92). What can be + // known about whether it will take, without writing anything, is asked now. + if o.pivots() { + if err := RuntimeWillTakeTheTrust(ctx, o, d.Run, say); err != nil { + return nil, err + } + } // 4. Can this machine reach what the bundle's images come from? // diff --git a/internal/bootstrap/trust.go b/internal/bootstrap/trust.go index 042aaba..ebe1efc 100644 --- a/internal/bootstrap/trust.go +++ b/internal/bootstrap/trust.go @@ -31,10 +31,23 @@ import ( // could hold it, because the retire step re-applies the bundle and takes away a carried record the // bundle does not name. The `docker` module, applied later, declares the artifact store's reach — // the mesh name on the node's port — finds it already in the file it has no record of, and takes it -// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). So nothing -// changes on the machine at the handover, no reload; from then on the module owns that member. The -// address the operator gave, when it is not the mesh name, stays as part of the machine's own -// configuration, as the hand-written trust it replaces would have. +// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member +// does not change at the handover. The module's first apply usually still changes the file and +// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`, +// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name, +// stays as part of the machine's own configuration, as the hand-written trust it replaces would have. +// +// **This rests on an assumption about order.** The module takes the mesh name's member over only +// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the +// network step (16), and once the node is on the network the controller composes the reach as +// `.:`. +// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:`, as it is before the +// network — its record would exist without the mesh name, and the member would stay the machine's +// for good: undeclaring the module would never take it out. Installing the node's seat holders +// earlier in genesis must revisit this. +// +// **The suffix** is read from this process's environment, and the controller reads its own: a mesh +// with a custom `MESH_INTERNAL_SUFFIX` sets it for both. // runtimeConfig is the container runtime's configuration, which the trust is written into. A // variable so a test writes into a file of its own. @@ -230,3 +243,75 @@ func inAny(ip net.IP, cidrs []string) bool { } return false } + +// procRoot is where a process's command line is read from. A variable so a test gives its own. +var procRoot = "/proc" + +// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known +// about it without writing anything, asked at step 1 so a machine that cannot take the trust is +// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92). +// +// - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to +// write over anything else (ADR 0102). +// - The runtime's service is known to systemd and active: the trust is taken by reloading it. +// - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both +// as a flag and in its file, so it would refuse the reload and keep distrusting the registry. +func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error { + names := RegistryNames(o) + if len(names) == 0 { + return nil + } + if raw, err := os.ReadFile(runtimeConfig); err == nil { + var object map[string]json.RawMessage + if len(strings.TrimSpace(string(raw))) > 0 { + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+ + "(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+ + "or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", ")) + } + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err) + } + + asking, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID") + cancel() + if err != nil { + return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+ + "reloading it; nothing has been changed", runtimeUnit, err) + } + props := map[string]string{} + for _, line := range strings.Split(out, "\n") { + if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok { + props[k] = v + } + } + if props["LoadState"] != "loaded" || props["ActiveState"] != "active" { + return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+ + "Genesis needs the runtime run by systemd as %s; nothing has been changed", + runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]), + strings.Join(names, ", "), runtimeUnit) + } + if pid := props["MainPID"]; pid != "" && pid != "0" { + if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil { + for _, arg := range strings.Split(string(cmdline), "\x00") { + if strings.HasPrefix(arg, "--insecure-registry") { + return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+ + "given both as a flag and in %s — so it would refuse the reload that makes it trust "+ + "%s. Move the flag's registries into the file and restart the runtime once; nothing "+ + "has been changed", arg, runtimeConfig, strings.Join(names, ", ")) + } + } + } + } + say(" registry trust " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded") + return nil +} + +func orUnknown(s string) string { + if s == "" { + return "unknown" + } + return s +} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go index 2c6ea09..8aecaa2 100644 --- a/internal/bootstrap/trust_test.go +++ b/internal/bootstrap/trust_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "os" "path/filepath" "strings" @@ -218,3 +219,129 @@ func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { t.Errorf("missing %v", missing) } } + +// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves +// as dockerd does: a push to a registry it has not been told to trust is refused with the error the +// first genesis met. The trust must be written and confirmed before anything is tagged or pushed; +// moving the trust after the push, or dropping it, fails this test. +func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { + o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`) + o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest) + o.Registry = "192.0.2.10:5100" + + pushed := false + mesh := aMeshThatAgrees(nil) + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "systemctl" || (name == "docker" && args[0] == "info") { + return runtime.run(ctx, name, args...) + } + runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " ")) + if name == "docker" { + switch args[0] { + case "tag": + return "", nil + case "push": + trusted := false + for _, n := range runtime.trusted { + trusted = trusted || strings.HasPrefix(args[1], n+"/") + } + if !trusted { + return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry) + } + pushed = true + return "", nil + case "image": + return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil + } + } + return mesh(name, args) + } + d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) { + if strings.HasSuffix(url, "/v2/") || pushed { + return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil + } + return http.StatusNotFound, "", nil + }} + var result Result + published, err := RegistryAndPublish(context.Background(), o, d, sys, + controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {}) + if err != nil { + t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands) + } + if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") { + t.Errorf("published as %q", published.Reference) + } + + firstPush, reloaded, confirmed := -1, -1, -1 + for i, c := range runtime.commands { + switch { + case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")): + firstPush = i + case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"): + reloaded = i + case strings.HasPrefix(c, "docker info"): + confirmed = i + } + } + if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) { + t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v", + reloaded, confirmed, firstPush, runtime.commands) + } + if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" { + t.Errorf("the result says it trusted %v", result.RegistryTrusted) + } +} + +// Preflight asks, without writing, what would stop the trust from taking at step 7. +func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) { + proc := t.TempDir() + was := procRoot + procRoot = proc + t.Cleanup(func() { procRoot = was }) + if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil { + t.Fatal(err) + } + unit := func(load, active string) Runner { + return func(_ context.Context, name string, args ...string) (string, error) { + return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil + } + } + cmdline := func(args ...string) { + if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil { + t.Fatal(err) + } + } + cmdline("/usr/bin/dockerd", "-H", "fd://") + + for _, c := range []struct { + name, file string + run Runner + flag bool + refused string + }{ + {"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""}, + {"no file yet", "", unit("loaded", "active"), false, ""}, + {"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"}, + {"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"}, + {"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"}, + } { + t.Run(c.name, func(t *testing.T) { + o, _, _ := trustSetup(t, c.file) + if c.flag { + cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000") + defer cmdline("/usr/bin/dockerd", "-H", "fd://") + } + before, _ := os.ReadFile(runtimeConfig) + err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {}) + switch { + case c.refused == "" && err != nil: + t.Errorf("refused: %v", err) + case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)): + t.Errorf("wanted a refusal naming %q, got %v", c.refused, err) + } + if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) { + t.Error("preflight wrote the runtime's file") + } + }) + } +}