package bootstrap import ( "context" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "time" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had // told the container runtime to trust that registry, so the push failed with "server gave HTTP // response to HTTPS client" — first by the address given, then by the mesh name the builder used, // and the trust the mesh wrote later named the registry's default port rather than the node's. // aRuntime is a container runtime on systemd whose registry configuration is what its file said at // its last reload: what a real dockerd does, and why a written file is not yet a trusted registry. type aRuntime struct { path string trusted []string commands []string // ignoresReload is a runtime started with options of its own, which a reload does not change. ignoresReload bool } func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) { line := name + " " + strings.Join(args, " ") r.commands = append(r.commands, line) switch { case name == "docker" && len(args) > 0 && args[0] == "info": index := map[string]map[string]any{"docker.io": {"Secure": true}} for _, n := range r.trusted { index[n] = map[string]any{"Name": n, "Secure": false} } out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index}) return string(out), nil case name == "systemctl" && strings.Contains(line, "reload"): if !r.ignoresReload { r.trusted = r.trusted[:0] raw, _ := os.ReadFile(r.path) var c struct { Insecure []string `json:"insecure-registries"` } _ = json.Unmarshal(raw, &c) r.trusted = append(r.trusted, c.Insecure...) } case strings.Contains(line, "is-enabled"): return "enabled", nil case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): return "LoadState=loaded\nActiveState=active\nSubState=running", nil } return "", nil } func (r *aRuntime) did(word string) int { n := 0 for _, c := range r.commands { if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) { n++ } } return n } func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) { t.Helper() restore := apply.ForTests(t.TempDir(), t.TempDir()) t.Cleanup(restore) dir := t.TempDir() was := runtimeConfig runtimeConfig = filepath.Join(dir, "daemon.json") t.Cleanup(func() { runtimeConfig = was }) if existing != "" { if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil { t.Fatal(err) } } t.Setenv(internalSuffixVar, "") sys, err := system.For("arch") if err != nil { t.Fatal(err) } o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100}, State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0} return o, &aRuntime{path: runtimeConfig}, sys } func insecureIn(t *testing.T) (string, map[string]json.RawMessage) { t.Helper() raw, err := os.ReadFile(runtimeConfig) if err != nil { t.Fatal(err) } var o map[string]json.RawMessage if err := json.Unmarshal(raw, &o); err != nil { t.Fatal(err) } var list []string _ = json.Unmarshal(o["insecure-registries"], &list) return fmt.Sprint(list), o } func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) { o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`) names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) if err != nil { t.Fatal(err) } // The address given and the mesh name the builder pushes by, both on the node's port. if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("trusted by %v", names) } list, object := insecureIn(t) if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("the runtime's list is %s", list) } if string(object["data-root"]) != `"/srv/docker"` { t.Errorf("the machine's own key was not kept: %s", object["data-root"]) } if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 { t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands) } // Re-run: nothing changes, nothing is reloaded. runtime.commands = nil if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { t.Fatal(err) } if runtime.did("reload") != 0 { t.Errorf("a re-run reloaded the runtime: %v", runtime.commands) } } // The port is the node's setting for the registry, not the module's default, whatever the operator // typed the address with — and loopback, which the runtime trusts already, is not written. func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) { o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}} t.Setenv(internalSuffixVar, "") if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" { t.Errorf("names %s", got) } o.Ports = FoundationPorts{} o.Registry = "192.0.2.10:5000" t.Setenv(internalSuffixVar, ".mesh.example") if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" { t.Errorf("names %s", got) } } // A runtime that still refuses after the write is refused here, with the names, before any push. func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) { o, runtime, sys := trustSetup(t, "") runtime.ignoresReload = true _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) if err == nil { t.Fatal("a runtime that does not trust the registry was let through to the push") } if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") || !strings.Contains(err.Error(), "Nothing has been pushed") { t.Errorf("the refusal does not say what: %v", err) } } // The handover: the `docker` module, applied later, writes the reach the controller composes for // the artifact store — the mesh name and the node's port. It finds it there: the file does not // change, the runtime is not reloaded, and the member it declares is recorded as the module's from // then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module // owns the trust it writes and taking the module away takes that member out. func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) { o, runtime, sys := trustSetup(t, `{"live-restore":true}`) if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { t.Fatal(err) } before, _ := os.ReadFile(runtimeConfig) module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[ {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json", "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"}, {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled", "reload-on":["docker.daemon"]}]}`, runtimeConfig))) if err != nil { t.Fatal(err) } runtime.commands = nil _, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared, runtime.run, nil, nil) if err != nil { t.Fatal(err) } after, _ := os.ReadFile(runtimeConfig) if string(before) != string(after) { t.Errorf("the handover changed the file:\n%s\n%s", before, after) } if runtime.did("reload") != 0 || runtime.did("restart") != 0 { t.Errorf("the handover touched the runtime: %v", runtime.commands) } if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 { t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into) } } func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`, []string{"10.42.0.1:5000", "anchor.internal:5000"}) if err != nil { t.Fatal(err) } if fmt.Sprint(missing) != "[anchor.internal:5000]" { t.Errorf("missing %v", missing) } }