jochen
41e640d668
Genesis pushed the control plane's image to the mesh's plain-HTTP registry before anything told the container runtime to trust it; the docker module that writes that trust is applied long after. Genesis now writes the trust itself at the registry step, with the same into-json writer and a reload, for the address given and the node's mesh name on the node's registry port, and refuses before any push unless the runtime says it trusts each. The docker module later finds the mesh name there and takes it over without a change.
221 lines
8.2 KiB
Go
221 lines
8.2 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/apply"
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
"github.com/novox/mesh-host/internal/system"
|
|
)
|
|
|
|
// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
|
|
// told the container runtime to trust that registry, so the push failed with "server gave HTTP
|
|
// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
|
|
// and the trust the mesh wrote later named the registry's default port rather than the node's.
|
|
|
|
// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
|
|
// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
|
|
type aRuntime struct {
|
|
path string
|
|
trusted []string
|
|
commands []string
|
|
// ignoresReload is a runtime started with options of its own, which a reload does not change.
|
|
ignoresReload bool
|
|
}
|
|
|
|
func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
|
|
line := name + " " + strings.Join(args, " ")
|
|
r.commands = append(r.commands, line)
|
|
switch {
|
|
case name == "docker" && len(args) > 0 && args[0] == "info":
|
|
index := map[string]map[string]any{"docker.io": {"Secure": true}}
|
|
for _, n := range r.trusted {
|
|
index[n] = map[string]any{"Name": n, "Secure": false}
|
|
}
|
|
out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
|
|
return string(out), nil
|
|
case name == "systemctl" && strings.Contains(line, "reload"):
|
|
if !r.ignoresReload {
|
|
r.trusted = r.trusted[:0]
|
|
raw, _ := os.ReadFile(r.path)
|
|
var c struct {
|
|
Insecure []string `json:"insecure-registries"`
|
|
}
|
|
_ = json.Unmarshal(raw, &c)
|
|
r.trusted = append(r.trusted, c.Insecure...)
|
|
}
|
|
case strings.Contains(line, "is-enabled"):
|
|
return "enabled", nil
|
|
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
|
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func (r *aRuntime) did(word string) int {
|
|
n := 0
|
|
for _, c := range r.commands {
|
|
if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
|
|
t.Helper()
|
|
restore := apply.ForTests(t.TempDir(), t.TempDir())
|
|
t.Cleanup(restore)
|
|
dir := t.TempDir()
|
|
was := runtimeConfig
|
|
runtimeConfig = filepath.Join(dir, "daemon.json")
|
|
t.Cleanup(func() { runtimeConfig = was })
|
|
if existing != "" {
|
|
if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
t.Setenv(internalSuffixVar, "")
|
|
sys, err := system.For("arch")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
|
|
State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
|
|
return o, &aRuntime{path: runtimeConfig}, sys
|
|
}
|
|
|
|
func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(runtimeConfig)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var o map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &o); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var list []string
|
|
_ = json.Unmarshal(o["insecure-registries"], &list)
|
|
return fmt.Sprint(list), o
|
|
}
|
|
|
|
func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
|
|
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)
|
|
|
|
names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The address given and the mesh name the builder pushes by, both on the node's port.
|
|
if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
|
t.Errorf("trusted by %v", names)
|
|
}
|
|
list, object := insecureIn(t)
|
|
if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
|
|
t.Errorf("the runtime's list is %s", list)
|
|
}
|
|
if string(object["data-root"]) != `"/srv/docker"` {
|
|
t.Errorf("the machine's own key was not kept: %s", object["data-root"])
|
|
}
|
|
if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
|
|
t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
|
|
}
|
|
|
|
// Re-run: nothing changes, nothing is reloaded.
|
|
runtime.commands = nil
|
|
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runtime.did("reload") != 0 {
|
|
t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
|
|
}
|
|
}
|
|
|
|
// The port is the node's setting for the registry, not the module's default, whatever the operator
|
|
// typed the address with — and loopback, which the runtime trusts already, is not written.
|
|
func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
|
|
o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
|
|
t.Setenv(internalSuffixVar, "")
|
|
if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
|
|
t.Errorf("names %s", got)
|
|
}
|
|
o.Ports = FoundationPorts{}
|
|
o.Registry = "192.0.2.10:5000"
|
|
t.Setenv(internalSuffixVar, ".mesh.example")
|
|
if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
|
|
t.Errorf("names %s", got)
|
|
}
|
|
}
|
|
|
|
// A runtime that still refuses after the write is refused here, with the names, before any push.
|
|
func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
|
|
o, runtime, sys := trustSetup(t, "")
|
|
runtime.ignoresReload = true
|
|
_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
|
if err == nil {
|
|
t.Fatal("a runtime that does not trust the registry was let through to the push")
|
|
}
|
|
if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
|
|
!strings.Contains(err.Error(), "Nothing has been pushed") {
|
|
t.Errorf("the refusal does not say what: %v", err)
|
|
}
|
|
}
|
|
|
|
// The handover: the `docker` module, applied later, writes the reach the controller composes for
|
|
// the artifact store — the mesh name and the node's port. It finds it there: the file does not
|
|
// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
|
|
// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
|
|
// owns the trust it writes and taking the module away takes that member out.
|
|
func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
|
|
o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
|
|
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, _ := os.ReadFile(runtimeConfig)
|
|
|
|
module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
|
|
{"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
|
|
"content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
|
|
{"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
|
|
"reload-on":["docker.daemon"]}]}`, runtimeConfig)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime.commands = nil
|
|
_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
|
|
runtime.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, _ := os.ReadFile(runtimeConfig)
|
|
if string(before) != string(after) {
|
|
t.Errorf("the handover changed the file:\n%s\n%s", before, after)
|
|
}
|
|
if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
|
|
t.Errorf("the handover touched the runtime: %v", runtime.commands)
|
|
}
|
|
if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
|
|
t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
|
|
}
|
|
}
|
|
|
|
func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
|
|
missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
|
|
[]string{"10.42.0.1:5000", "anchor.internal:5000"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fmt.Sprint(missing) != "[anchor.internal:5000]" {
|
|
t.Errorf("missing %v", missing)
|
|
}
|
|
}
|