From 9e233e1d87569273b4ed353c450b0bd27d21039b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 02:57:35 +0200 Subject: [PATCH] Replay R92's commit before its fix: mesh-host main without genesis's registry trust (#91 reverted) Never to be merged. The replay of hq issue 92 runs genesis at the commit before its fix, and the fix's own first parent stops at step 6 on issue 146; this is today's genesis without #91, so the bed reaches the push it replays (mesh-lab replays/register.go R92). --- internal/bootstrap/bootstrap.go | 82 +++----- internal/bootstrap/phase3.go | 8 + internal/bootstrap/preflight.go | 7 - internal/bootstrap/trust.go | 317 ---------------------------- internal/bootstrap/trust_test.go | 347 ------------------------------- 5 files changed, 34 insertions(+), 727 deletions(-) delete mode 100644 internal/bootstrap/trust.go delete mode 100644 internal/bootstrap/trust_test.go diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index c047eb1..d1e5c0f 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -36,7 +36,6 @@ import ( "time" "github.com/novox/mesh-host/internal/firewall" - "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/tunnel" ) @@ -293,14 +292,12 @@ type Result struct { Agent string `json:"host-agent,omitempty"` // Registry is the mesh's own artifact store, once it answers. - Registry string `json:"registry,omitempty"` - RegistryReplied int `json:"registry-replied,omitempty"` - RegistryKnown bool `json:"registry-already-registered,omitempty"` - RegistryRunning string `json:"registry-container,omitempty"` - // RegistryTrusted are the names the runtime was told to trust the registry by, before the push. - RegistryTrusted []string `json:"registry-trusted,omitempty"` - PublishedAs string `json:"control-plane-image,omitempty"` - PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` // Permanent is the control plane as an ordinary module. Permanent string `json:"permanent-container,omitempty"` @@ -558,9 +555,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // Named rather than attempted. Everything from step 6 on is a conversation with a // control plane that a dry run has not raised, so there is nothing to ask and nothing // honest to report about the answers. - say(" would then enrol " + o.Node + ", have the runtime trust the registry as " + - strings.Join(RegistryNames(o), ", ") + ",") - say(" install the registry from " + + say(" would then enrol " + o.Node + ", install the registry from " + o.Catalogue + ", push the control plane's image into it,") say(" reinstall the control plane as a module, and drop " + rewritten.TempName) @@ -659,10 +654,26 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepOperator, err) } - // ---- 7. registry, 8. publish ----------------------------------------------------------- - published, err := RegistryAndPublish(ctx, o, d, sys, temporary, controlPlaneImage, &result, say) + // ---- 7. registry ---------------------------------------------------------------------- + say("registry — somewhere for this mesh to keep its own images") + registry, err := InstallRegistry(ctx, o, d, temporary, say) + result.Registry, result.RegistryRunning = registry.Address, registry.Container + result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered if err != nil { - return result, err + return result, failed(StepRegistry, err) + } + + // ---- 8. publish ----------------------------------------------------------------------- + say("publish — the control plane's image gets its first manifest digest") + // **The image this mesh built, not the one the installer carried.** The carried one is the + // builder; publishing it here would put a builder in the registry under the control plane's + // name and install it as the control plane — which is exactly what happened the first time + // this ran, and presented as a control plane that started, printed a builder's usage, and + // exited cleanly over and over. + published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say) + result.PublishedAs, result.PublishedAlready = published.Reference, published.Already + if err != nil { + return result, failed(StepPublish, err) } // ---- 9. control plane ----------------------------------------------------------------- @@ -792,44 +803,3 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, nil } - -// RegistryAndPublish is steps 7 and 8: the runtime trusts the mesh's registry, the registry is -// installed and answers, and the control plane's image is pushed into it. -// -// **One function so the order is held by a test** (novox/hq issue 92). The runtime refuses plain -// HTTP to a registry it has not been told to trust, and the module that tells it is applied long -// after the push — so the trust is written and confirmed here, before anything is tagged or pushed. -// A refusal names the step it happened in. -func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.System, temporary controlPlane, - controlPlaneImage string, result *Result, say func(string)) (Published, error) { - - // ---- 7. registry ---------------------------------------------------------------------- - say("registry — somewhere for this mesh to keep its own images") - // Trusted by the runtime before anything is pushed to it, by every name the mesh will use for - // it here: the module that writes this trust is applied long after the push (novox/hq issue 92). - trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say) - result.RegistryTrusted = trusted - if err != nil { - return Published{}, failed(StepRegistry, err) - } - registry, err := InstallRegistry(ctx, o, d, temporary, say) - result.Registry, result.RegistryRunning = registry.Address, registry.Container - result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered - if err != nil { - return Published{}, failed(StepRegistry, err) - } - - // ---- 8. publish ----------------------------------------------------------------------- - say("publish — the control plane's image gets its first manifest digest") - // **The image this mesh built, not the one the installer carried.** The carried one is the - // builder; publishing it here would put a builder in the registry under the control plane's - // name and install it as the control plane — which is exactly what happened the first time - // this ran, and presented as a control plane that started, printed a builder's usage, and - // exited cleanly over and over. - published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say) - result.PublishedAs, result.PublishedAlready = published.Reference, published.Already - if err != nil { - return published, failed(StepPublish, err) - } - return published, nil -} diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 216bc88..5049227 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -478,3 +478,11 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu "store with. Its server container has to carry the name the foundation raised", module, store.Name) } + +// orUnknown is s, or a word for nothing: trust.go held it, and this branch takes trust.go out. +func orUnknown(s string) string { + if strings.TrimSpace(s) == "" { + return "(nothing)" + } + return s +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index 0e4ca46..74f2164 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -127,13 +127,6 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil { return nil, err } - // The registry's trust is written at step 7, before the push (novox/hq issue 92). What can be - // known about whether it will take, without writing anything, is asked now. - if o.pivots() { - if err := RuntimeWillTakeTheTrust(ctx, o, d.Run, say); err != nil { - return nil, err - } - } // 4. Can this machine reach what the bundle's images come from? // diff --git a/internal/bootstrap/trust.go b/internal/bootstrap/trust.go deleted file mode 100644 index ebe1efc..0000000 --- a/internal/bootstrap/trust.go +++ /dev/null @@ -1,317 +0,0 @@ -package bootstrap - -import ( - "context" - "encoding/json" - "fmt" - "net" - "os" - "strconv" - "strings" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92). -// -// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private -// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry -// it has not been told to trust, and the module that tells it — `docker`, which writes the -// artifact store's reach into the runtime's configuration — is applied long after genesis pushes -// the control plane's image. So genesis writes that trust itself, first, with the same writer the -// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and -// never restarted. It then asks the runtime whether it will speak to every name, and refuses with -// the runtime's answer before anything is pushed. -// -// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation -// could hold it, because the retire step re-applies the bundle and takes away a carried record the -// bundle does not name. The `docker` module, applied later, declares the artifact store's reach — -// the mesh name on the node's port — finds it already in the file it has no record of, and takes it -// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member -// does not change at the handover. The module's first apply usually still changes the file and -// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`, -// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name, -// stays as part of the machine's own configuration, as the hand-written trust it replaces would have. -// -// **This rests on an assumption about order.** The module takes the mesh name's member over only -// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the -// network step (16), and once the node is on the network the controller composes the reach as -// `.:`. -// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:`, as it is before the -// network — its record would exist without the mesh name, and the member would stay the machine's -// for good: undeclaring the module would never take it out. Installing the node's seat holders -// earlier in genesis must revisit this. -// -// **The suffix** is read from this process's environment, and the controller reads its own: a mesh -// with a custom `MESH_INTERNAL_SUFFIX` sets it for both. - -// runtimeConfig is the container runtime's configuration, which the trust is written into. A -// variable so a test writes into a file of its own. -var runtimeConfig = "/etc/docker/daemon.json" - -// runtimeUnit is the runtime's service, reloaded when the file changed. -const runtimeUnit = "docker.service" - -// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable -// and default the controller composes `.` from (mesh-controller internal/overlay), -// which is the name the artifact store's reach — and so every push by the mesh's builder — uses. -const ( - internalSuffixVar = "MESH_INTERNAL_SUFFIX" - defaultInternalSuffix = "internal" -) - -// RegistryNames are the names the mesh will use for its registry on this node, each as host:port. -// -// The address the operator gave, and the node's mesh name: the controller's reach for the artifact -// store, which the builder pushes to without being told. Both on the port the registry module -// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A -// loopback name is left out: the runtime trusts loopback without being told, and the mesh never -// names it to another node. -func RegistryNames(o Options) []string { - port := strconv.Itoa(o.Ports.orDefaults().Registry) - var names []string - add := func(hostport string) { - host, _, err := net.SplitHostPort(hostport) - if err != nil || host == "" || isLoopback(host) { - return - } - for _, n := range names { - if n == hostport { - return - } - } - names = append(names, hostport) - } - if host, _, err := net.SplitHostPort(o.Registry); err == nil { - add(net.JoinHostPort(strings.Trim(host, "[]"), port)) - } - if node := strings.TrimSpace(o.Node); node != "" { - add(net.JoinHostPort(node+"."+internalSuffix(), port)) - } - return names -} - -func internalSuffix() string { - if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" { - return strings.TrimPrefix(v, ".") - } - return defaultInternalSuffix -} - -func isLoopback(host string) bool { - if strings.EqualFold(host, "localhost") { - return true - } - ip := net.ParseIP(strings.Trim(host, "[]")) - return ip != nil && ip.IsLoopback() -} - -// trustDeclaration is the declaration genesis applies to write the trust: the same two resources -// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it. -func trustDeclaration(names []string) (*declaration.Declaration, error) { - keys, err := json.Marshal(map[string][]string{"insecure-registries": names}) - if err != nil { - return nil, err - } - raw, err := json.Marshal(map[string]any{ - "declaration": 1, - "resources": []map[string]any{ - {"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644", - "into": "json", "content": string(keys) + "\n"}, - {"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit, - "state": "running", "reload-on": []string{"genesis.registry-trust"}}, - }, - }) - if err != nil { - return nil, err - } - return declaration.ParseTrusted(raw) -} - -// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here, -// then refuses unless the runtime says it will speak plain HTTP to each. -func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner, - say func(string)) ([]string, error) { - - names := RegistryNames(o) - if len(names) == 0 { - say(" trusted only loopback names this registry, which the runtime trusts already") - return nil, nil - } - d, err := trustDeclaration(names) - if err != nil { - return names, err - } - - // One apply at a time on this machine, as ApplyBundle does: a host already running here may be - // writing the same file. - unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") }) - if err != nil { - return names, err - } - report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run, - func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) - unlock() - if err != nil { - return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+ - "The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+ - "runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err) - } - if report.Changed() { - say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig + - ", the runtime reloaded") - } else { - say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so") - } - return names, runtimeWillSpeakTo(ctx, o, run, names) -} - -// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out. -// -// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on -// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error. -func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error { - deadline := time.Now().Add(o.Wait) - var missing []string - var last error - for { - asking, cancel := context.WithTimeout(ctx, o.Timeout) - out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}") - cancel() - last = err - if err == nil { - missing, last = untrusted(out, names) - if last == nil && len(missing) == 0 { - return nil - } - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(answerEvery): - } - } - if last != nil { - return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last) - } - return fmt.Errorf( - "the container runtime does not trust %s after %s says it should, so it would refuse to push to "+ - "it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+ - "with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+ - "Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit) -} - -// untrusted reads the runtime's registry configuration and gives back the names it would speak -// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in -// a range the runtime trusts. -func untrusted(registryConfig string, names []string) ([]string, error) { - var c struct { - InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"` - IndexConfigs map[string]struct { - Secure bool `json:"Secure"` - } `json:"IndexConfigs"` - } - if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil { - return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err) - } - var missing []string - for _, name := range names { - if index, ok := c.IndexConfigs[name]; ok && !index.Secure { - continue - } - host, _, _ := net.SplitHostPort(name) - if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) { - continue - } - missing = append(missing, name) - } - return missing, nil -} - -func inAny(ip net.IP, cidrs []string) bool { - for _, cidr := range cidrs { - if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) { - return true - } - } - return false -} - -// procRoot is where a process's command line is read from. A variable so a test gives its own. -var procRoot = "/proc" - -// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known -// about it without writing anything, asked at step 1 so a machine that cannot take the trust is -// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92). -// -// - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to -// write over anything else (ADR 0102). -// - The runtime's service is known to systemd and active: the trust is taken by reloading it. -// - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both -// as a flag and in its file, so it would refuse the reload and keep distrusting the registry. -func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error { - names := RegistryNames(o) - if len(names) == 0 { - return nil - } - if raw, err := os.ReadFile(runtimeConfig); err == nil { - var object map[string]json.RawMessage - if len(strings.TrimSpace(string(raw))) > 0 { - if err := json.Unmarshal(raw, &object); err != nil || object == nil { - return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+ - "(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+ - "or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", ")) - } - } - } else if !os.IsNotExist(err) { - return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err) - } - - asking, cancel := context.WithTimeout(ctx, o.Timeout) - out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID") - cancel() - if err != nil { - return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+ - "reloading it; nothing has been changed", runtimeUnit, err) - } - props := map[string]string{} - for _, line := range strings.Split(out, "\n") { - if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok { - props[k] = v - } - } - if props["LoadState"] != "loaded" || props["ActiveState"] != "active" { - return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+ - "Genesis needs the runtime run by systemd as %s; nothing has been changed", - runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]), - strings.Join(names, ", "), runtimeUnit) - } - if pid := props["MainPID"]; pid != "" && pid != "0" { - if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil { - for _, arg := range strings.Split(string(cmdline), "\x00") { - if strings.HasPrefix(arg, "--insecure-registry") { - return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+ - "given both as a flag and in %s — so it would refuse the reload that makes it trust "+ - "%s. Move the flag's registries into the file and restart the runtime once; nothing "+ - "has been changed", arg, runtimeConfig, strings.Join(names, ", ")) - } - } - } - } - say(" registry trust " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded") - return nil -} - -func orUnknown(s string) string { - if s == "" { - return "unknown" - } - return s -} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go deleted file mode 100644 index 8aecaa2..0000000 --- a/internal/bootstrap/trust_test.go +++ /dev/null @@ -1,347 +0,0 @@ -package bootstrap - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had -// told the container runtime to trust that registry, so the push failed with "server gave HTTP -// response to HTTPS client" — first by the address given, then by the mesh name the builder used, -// and the trust the mesh wrote later named the registry's default port rather than the node's. - -// aRuntime is a container runtime on systemd whose registry configuration is what its file said at -// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry. -type aRuntime struct { - path string - trusted []string - commands []string - // ignoresReload is a runtime started with options of its own, which a reload does not change. - ignoresReload bool -} - -func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) { - line := name + " " + strings.Join(args, " ") - r.commands = append(r.commands, line) - switch { - case name == "docker" && len(args) > 0 && args[0] == "info": - index := map[string]map[string]any{"docker.io": {"Secure": true}} - for _, n := range r.trusted { - index[n] = map[string]any{"Name": n, "Secure": false} - } - out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index}) - return string(out), nil - case name == "systemctl" && strings.Contains(line, "reload"): - if !r.ignoresReload { - r.trusted = r.trusted[:0] - raw, _ := os.ReadFile(r.path) - var c struct { - Insecure []string `json:"insecure-registries"` - } - _ = json.Unmarshal(raw, &c) - r.trusted = append(r.trusted, c.Insecure...) - } - case strings.Contains(line, "is-enabled"): - return "enabled", nil - case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): - return "LoadState=loaded\nActiveState=active\nSubState=running", nil - } - return "", nil -} - -func (r *aRuntime) did(word string) int { - n := 0 - for _, c := range r.commands { - if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) { - n++ - } - } - return n -} - -func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) { - t.Helper() - restore := apply.ForTests(t.TempDir(), t.TempDir()) - t.Cleanup(restore) - dir := t.TempDir() - was := runtimeConfig - runtimeConfig = filepath.Join(dir, "daemon.json") - t.Cleanup(func() { runtimeConfig = was }) - if existing != "" { - if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil { - t.Fatal(err) - } - } - t.Setenv(internalSuffixVar, "") - sys, err := system.For("arch") - if err != nil { - t.Fatal(err) - } - o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100}, - State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0} - return o, &aRuntime{path: runtimeConfig}, sys -} - -func insecureIn(t *testing.T) (string, map[string]json.RawMessage) { - t.Helper() - raw, err := os.ReadFile(runtimeConfig) - if err != nil { - t.Fatal(err) - } - var o map[string]json.RawMessage - if err := json.Unmarshal(raw, &o); err != nil { - t.Fatal(err) - } - var list []string - _ = json.Unmarshal(o["insecure-registries"], &list) - return fmt.Sprint(list), o -} - -func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) { - o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`) - - names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) - if err != nil { - t.Fatal(err) - } - // The address given and the mesh name the builder pushes by, both on the node's port. - if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" { - t.Errorf("trusted by %v", names) - } - list, object := insecureIn(t) - if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" { - t.Errorf("the runtime's list is %s", list) - } - if string(object["data-root"]) != `"/srv/docker"` { - t.Errorf("the machine's own key was not kept: %s", object["data-root"]) - } - if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 { - t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands) - } - - // Re-run: nothing changes, nothing is reloaded. - runtime.commands = nil - if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { - t.Fatal(err) - } - if runtime.did("reload") != 0 { - t.Errorf("a re-run reloaded the runtime: %v", runtime.commands) - } -} - -// The port is the node's setting for the registry, not the module's default, whatever the operator -// typed the address with — and loopback, which the runtime trusts already, is not written. -func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) { - o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}} - t.Setenv(internalSuffixVar, "") - if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" { - t.Errorf("names %s", got) - } - o.Ports = FoundationPorts{} - o.Registry = "192.0.2.10:5000" - t.Setenv(internalSuffixVar, ".mesh.example") - if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" { - t.Errorf("names %s", got) - } -} - -// A runtime that still refuses after the write is refused here, with the names, before any push. -func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) { - o, runtime, sys := trustSetup(t, "") - runtime.ignoresReload = true - _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}) - if err == nil { - t.Fatal("a runtime that does not trust the registry was let through to the push") - } - if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") || - !strings.Contains(err.Error(), "Nothing has been pushed") { - t.Errorf("the refusal does not say what: %v", err) - } -} - -// The handover: the `docker` module, applied later, writes the reach the controller composes for -// the artifact store — the mesh name and the node's port. It finds it there: the file does not -// change, the runtime is not reloaded, and the member it declares is recorded as the module's from -// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module -// owns the trust it writes and taking the module away takes that member out. -func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) { - o, runtime, sys := trustSetup(t, `{"live-restore":true}`) - if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil { - t.Fatal(err) - } - before, _ := os.ReadFile(runtimeConfig) - - module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[ - {"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json", - "content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"}, - {"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled", - "reload-on":["docker.daemon"]}]}`, runtimeConfig))) - if err != nil { - t.Fatal(err) - } - runtime.commands = nil - _, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared, - runtime.run, nil, nil) - if err != nil { - t.Fatal(err) - } - after, _ := os.ReadFile(runtimeConfig) - if string(before) != string(after) { - t.Errorf("the handover changed the file:\n%s\n%s", before, after) - } - if runtime.did("reload") != 0 || runtime.did("restart") != 0 { - t.Errorf("the handover touched the runtime: %v", runtime.commands) - } - if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 { - t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into) - } -} - -func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { - missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`, - []string{"10.42.0.1:5000", "anchor.internal:5000"}) - if err != nil { - t.Fatal(err) - } - if fmt.Sprint(missing) != "[anchor.internal:5000]" { - t.Errorf("missing %v", missing) - } -} - -// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves -// as dockerd does: a push to a registry it has not been told to trust is refused with the error the -// first genesis met. The trust must be written and confirmed before anything is tagged or pushed; -// moving the trust after the push, or dropping it, fails this test. -func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { - o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`) - o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest) - o.Registry = "192.0.2.10:5100" - - pushed := false - mesh := aMeshThatAgrees(nil) - run := func(ctx context.Context, name string, args ...string) (string, error) { - if name == "systemctl" || (name == "docker" && args[0] == "info") { - return runtime.run(ctx, name, args...) - } - runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " ")) - if name == "docker" { - switch args[0] { - case "tag": - return "", nil - case "push": - trusted := false - for _, n := range runtime.trusted { - trusted = trusted || strings.HasPrefix(args[1], n+"/") - } - if !trusted { - return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry) - } - pushed = true - return "", nil - case "image": - return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil - } - } - return mesh(name, args) - } - d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) { - if strings.HasSuffix(url, "/v2/") || pushed { - return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil - } - return http.StatusNotFound, "", nil - }} - var result Result - published, err := RegistryAndPublish(context.Background(), o, d, sys, - controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {}) - if err != nil { - t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands) - } - if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") { - t.Errorf("published as %q", published.Reference) - } - - firstPush, reloaded, confirmed := -1, -1, -1 - for i, c := range runtime.commands { - switch { - case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")): - firstPush = i - case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"): - reloaded = i - case strings.HasPrefix(c, "docker info"): - confirmed = i - } - } - if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) { - t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v", - reloaded, confirmed, firstPush, runtime.commands) - } - if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" { - t.Errorf("the result says it trusted %v", result.RegistryTrusted) - } -} - -// Preflight asks, without writing, what would stop the trust from taking at step 7. -func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) { - proc := t.TempDir() - was := procRoot - procRoot = proc - t.Cleanup(func() { procRoot = was }) - if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil { - t.Fatal(err) - } - unit := func(load, active string) Runner { - return func(_ context.Context, name string, args ...string) (string, error) { - return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil - } - } - cmdline := func(args ...string) { - if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil { - t.Fatal(err) - } - } - cmdline("/usr/bin/dockerd", "-H", "fd://") - - for _, c := range []struct { - name, file string - run Runner - flag bool - refused string - }{ - {"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""}, - {"no file yet", "", unit("loaded", "active"), false, ""}, - {"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"}, - {"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"}, - {"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"}, - } { - t.Run(c.name, func(t *testing.T) { - o, _, _ := trustSetup(t, c.file) - if c.flag { - cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000") - defer cmdline("/usr/bin/dockerd", "-H", "fd://") - } - before, _ := os.ReadFile(runtimeConfig) - err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {}) - switch { - case c.refused == "" && err != nil: - t.Errorf("refused: %v", err) - case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)): - t.Errorf("wanted a refusal naming %q, got %v", c.refused, err) - } - if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) { - t.Error("preflight wrote the runtime's file") - } - }) - } -}