package bootstrap import ( "context" "encoding/json" "fmt" "net/http" "os" "path/filepath" "strings" "testing" "time" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had // told the container runtime to trust that registry, so the push failed with "server gave HTTP // response to HTTPS client" — first by the address given, then by the mesh name the builder used, // and the trust the mesh wrote later named the registry's default port rather than the node's. // aRuntime is a container runtime on systemd whose registry configuration is what its file said at // its last reload: what a real dockerd does, and why a written file is not yet a trusted registry. type aRuntime struct { path string trusted []string commands []string // ignoresReload is a runtime started with options of its own, which a reload does not change. ignoresReload bool } func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) { line := name + " " + strings.Join(args, " ") r.commands = append(r.commands, line) switch { case name == "docker" && len(args) > 0 && args[0] == "info": index := map[string]map[string]any{"docker.io": {"Secure": true}} for _, n := range r.trusted { index[n] = map[string]any{"Name": n, "Secure": false} } out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index}) return string(out), nil case name == "systemctl" && strings.Contains(line, "reload"): if !r.ignoresReload { r.trusted = r.trusted[:0] raw, _ := os.ReadFile(r.path) var c struct { Insecure []string `json:"insecure-registries"` } _ = json.Unmarshal(raw, &c) r.trusted = append(r.trusted, c.Insecure...) } case strings.Contains(line, "is-enabled"): return "enabled", nil case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): return "LoadState=loaded\nActiveState=active\nSubState=running", nil } return "", nil } func (r *aRuntime) did(word string) int { n := 0 for _, c := range r.commands { if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) { n++ } } return n } func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) { t.Helper() restore := apply.ForTests(t.TempDir(), t.TempDir()) t.Cleanup(restore) dir := t.TempDir() was := runtimeConfig runtimeConfig = filepath.Join(dir, "daemon.json") t.Cleanup(func() { runtimeConfig = was }) if existing != "" { if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil { t.Fatal(err) } } t.Setenv(internalSuffixVar, "") sys, err := system.For("arch") if err != nil { t.Fatal(err) } o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100}, State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0} return o, &aRuntime{path: runtimeConfig}, sys } func insecureIn(t *testing.T) (string, map[string]json.RawMessage) { t.Helper() raw, err := os.ReadFile(runtimeConfig) if err != nil { t.Fatal(err) } var o map[string]json.RawMessage if err := json.Unmarshal(raw, &o); err != nil { t.Fatal(err) } var list []string _ = json.Unmarshal(o["insecure-registries"], &list) return fmt.Sprint(list), o } func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) { o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`) names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) if err != nil { t.Fatal(err) } // The address given and the mesh name the builder pushes by, both on the node's port. if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("trusted by %v", names) } list, object := insecureIn(t) if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("the runtime's list is %s", list) } if string(object["data-root"]) != `"/srv/docker"` { t.Errorf("the machine's own key was not kept: %s", object["data-root"]) } if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 { t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands) } // Re-run: nothing changes, nothing is reloaded. runtime.commands = nil if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { t.Fatal(err) } if runtime.did("reload") != 0 { t.Errorf("a re-run reloaded the runtime: %v", runtime.commands) } } // The port is the node's setting for the registry, not the module's default, whatever the operator // typed the address with — and loopback, which the runtime trusts already, is not written. func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) { o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}} t.Setenv(internalSuffixVar, "") if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" { t.Errorf("names %s", got) } o.Ports = FoundationPorts{} o.Registry = "192.0.2.10:5000" t.Setenv(internalSuffixVar, ".mesh.example") if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" { t.Errorf("names %s", got) } } // A runtime that still refuses after the write is refused here, with the names, before any push. func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) { o, runtime, sys := trustSetup(t, "") runtime.ignoresReload = true _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) if err == nil { t.Fatal("a runtime that does not trust the registry was let through to the push") } if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") || !strings.Contains(err.Error(), "Nothing has been pushed") { t.Errorf("the refusal does not say what: %v", err) } } // The handover: the `docker` module, applied later, writes the reach the controller composes for // the artifact store — the mesh name and the node's port. It finds it there: the file does not // change, the runtime is not reloaded, and the member it declares is recorded as the module's from // then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module // owns the trust it writes and taking the module away takes that member out. func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) { o, runtime, sys := trustSetup(t, `{"live-restore":true}`) if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { t.Fatal(err) } before, _ := os.ReadFile(runtimeConfig) module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json", "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"}, {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled", "reload-on":["docker.daemon"]}]}`, runtimeConfig))) if err != nil { t.Fatal(err) } runtime.commands = nil _, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared, runtime.run, nil, nil) if err != nil { t.Fatal(err) } after, _ := os.ReadFile(runtimeConfig) if string(before) != string(after) { t.Errorf("the handover changed the file:\n%s\n%s", before, after) } if runtime.did("reload") != 0 || runtime.did("restart") != 0 { t.Errorf("the handover touched the runtime: %v", runtime.commands) } if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 { t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into) } } func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`, []string{"10.42.0.1:5000", "anchor.internal:5000"}) if err != nil { t.Fatal(err) } if fmt.Sprint(missing) != "[anchor.internal:5000]" { t.Errorf("missing %v", missing) } } // **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves // as dockerd does: a push to a registry it has not been told to trust is refused with the error the // first genesis met. The trust must be written and confirmed before anything is tagged or pushed; // moving the trust after the push, or dropping it, fails this test. func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`) o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest) o.Registry = "192.0.2.10:5100" pushed := false mesh := aMeshThatAgrees(nil) run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "systemctl" || (name == "docker" && args[0] == "info") { return runtime.run(ctx, name, args...) } runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " ")) if name == "docker" { switch args[0] { case "tag": return "", nil case "push": trusted := false for _, n := range runtime.trusted { trusted = trusted || strings.HasPrefix(args[1], n+"/") } if !trusted { return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry) } pushed = true return "", nil case "image": return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil } } return mesh(name, args) } d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) { if strings.HasSuffix(url, "/v2/") || pushed { return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil } return http.StatusNotFound, "", nil }} var result Result published, err := RegistryAndPublish(context.Background(), o, d, sys, controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {}) if err != nil { t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands) } if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") { t.Errorf("published as %q", published.Reference) } firstPush, reloaded, confirmed := -1, -1, -1 for i, c := range runtime.commands { switch { case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")): firstPush = i case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"): reloaded = i case strings.HasPrefix(c, "docker info"): confirmed = i } } if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) { t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v", reloaded, confirmed, firstPush, runtime.commands) } if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("the result says it trusted %v", result.RegistryTrusted) } } // Preflight asks, without writing, what would stop the trust from taking at step 7. func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) { proc := t.TempDir() was := procRoot procRoot = proc t.Cleanup(func() { procRoot = was }) if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil { t.Fatal(err) } unit := func(load, active string) Runner { return func(_ context.Context, name string, args ...string) (string, error) { return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil } } cmdline := func(args ...string) { if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil { t.Fatal(err) } } cmdline("/usr/bin/dockerd", "-H", "fd://") for _, c := range []struct { name, file string run Runner flag bool refused string }{ {"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""}, {"no file yet", "", unit("loaded", "active"), false, ""}, {"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"}, {"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"}, {"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"}, } { t.Run(c.name, func(t *testing.T) { o, _, _ := trustSetup(t, c.file) if c.flag { cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000") defer cmdline("/usr/bin/dockerd", "-H", "fd://") } before, _ := os.ReadFile(runtimeConfig) err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {}) switch { case c.refused == "" && err != nil: t.Errorf("refused: %v", err) case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)): t.Errorf("wanted a refusal naming %q, got %v", c.refused, err) } if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) { t.Error("preflight wrote the runtime's file") } }) } }