novox/mesh-host
7
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request 'Genesis trusts its registry before it pushes to it (issue 92)' (#91) from fix/92-genesis-trusts-its-registry into main

Browse Source
This commit was merged in pull request #91.
This commit is contained in:
mesh-admin
2026-10-12 14:36:16 +00:00
parent a1f9856b39 0818e4c29a
commit dd9ab7927d
4 changed files with 727 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/bootstrap/bootstrap.go
+56 -26
View File
@@ -36,6 +36,7 @@ import (
"time"
"github.com/novox/mesh-host/internal/firewall"
"github.com/novox/mesh-host/internal/system"
"github.com/novox/mesh-host/internal/tunnel"
)
@@ -292,12 +293,14 @@ type Result struct {
Agent string `json:"host-agent,omitempty"`
// Registry is the mesh's own artifact store, once it answers.
Registry string `json:"registry,omitempty"`
RegistryReplied int `json:"registry-replied,omitempty"`
RegistryKnown bool `json:"registry-already-registered,omitempty"`
RegistryRunning string `json:"registry-container,omitempty"`
PublishedAs string `json:"control-plane-image,omitempty"`
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
Registry string `json:"registry,omitempty"`
RegistryReplied int `json:"registry-replied,omitempty"`
RegistryKnown bool `json:"registry-already-registered,omitempty"`
RegistryRunning string `json:"registry-container,omitempty"`
// RegistryTrusted are the names the runtime was told to trust the registry by, before the push.
RegistryTrusted []string `json:"registry-trusted,omitempty"`
PublishedAs string `json:"control-plane-image,omitempty"`
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
// Permanent is the control plane as an ordinary module.
Permanent string `json:"permanent-container,omitempty"`
@@ -555,7 +558,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
// Named rather than attempted. Everything from step 6 on is a conversation with a
// control plane that a dry run has not raised, so there is nothing to ask and nothing
// honest to report about the answers.
say(" would then enrol " + o.Node + ", install the registry from " +
say(" would then enrol " + o.Node + ", have the runtime trust the registry as " +
strings.Join(RegistryNames(o), ", ") + ",")
say(" install the registry from " +
o.Catalogue + ", push the control plane's image into it,")
say(" reinstall the control plane as a module, and drop " +
rewritten.TempName)
@@ -654,26 +659,10 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
return result, failed(StepOperator, err)
}
// ---- 7. registry ----------------------------------------------------------------------
say("registry — somewhere for this mesh to keep its own images")
registry, err := InstallRegistry(ctx, o, d, temporary, say)
result.Registry, result.RegistryRunning = registry.Address, registry.Container
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
// ---- 7. registry, 8. publish -----------------------------------------------------------
published, err := RegistryAndPublish(ctx, o, d, sys, temporary, controlPlaneImage, &result, say)
if err != nil {
return result, failed(StepRegistry, err)
}
// ---- 8. publish -----------------------------------------------------------------------
say("publish — the control plane's image gets its first manifest digest")
// **The image this mesh built, not the one the installer carried.** The carried one is the
// builder; publishing it here would put a builder in the registry under the control plane's
// name and install it as the control plane — which is exactly what happened the first time
// this ran, and presented as a control plane that started, printed a builder's usage, and
// exited cleanly over and over.
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
if err != nil {
return result, failed(StepPublish, err)
return result, err
}
// ---- 9. control plane -----------------------------------------------------------------
@@ -803,3 +792,44 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
return result, nil
}
// RegistryAndPublish is steps 7 and 8: the runtime trusts the mesh's registry, the registry is
// installed and answers, and the control plane's image is pushed into it.
//
// **One function so the order is held by a test** (novox/hq issue 92). The runtime refuses plain
// HTTP to a registry it has not been told to trust, and the module that tells it is applied long
// after the push — so the trust is written and confirmed here, before anything is tagged or pushed.
// A refusal names the step it happened in.
func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.System, temporary controlPlane,
controlPlaneImage string, result *Result, say func(string)) (Published, error) {
// ---- 7. registry ----------------------------------------------------------------------
say("registry — somewhere for this mesh to keep its own images")
// Trusted by the runtime before anything is pushed to it, by every name the mesh will use for
// it here: the module that writes this trust is applied long after the push (novox/hq issue 92).
trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say)
result.RegistryTrusted = trusted
if err != nil {
return Published{}, failed(StepRegistry, err)
}
registry, err := InstallRegistry(ctx, o, d, temporary, say)
result.Registry, result.RegistryRunning = registry.Address, registry.Container
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
if err != nil {
return Published{}, failed(StepRegistry, err)
}
// ---- 8. publish -----------------------------------------------------------------------
say("publish — the control plane's image gets its first manifest digest")
// **The image this mesh built, not the one the installer carried.** The carried one is the
// builder; publishing it here would put a builder in the registry under the control plane's
// name and install it as the control plane — which is exactly what happened the first time
// this ran, and presented as a control plane that started, printed a builder's usage, and
// exited cleanly over and over.
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
if err != nil {
return published, failed(StepPublish, err)
}
return published, nil
}
internal/bootstrap/preflight.go
+7
View File
@@ -110,6 +110,13 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil {
return nil, err
}
// The registry's trust is written at step 7, before the push (novox/hq issue 92). What can be
// known about whether it will take, without writing anything, is asked now.
if o.pivots() {
if err := RuntimeWillTakeTheTrust(ctx, o, d.Run, say); err != nil {
return nil, err
}
}
// 4. Can this machine reach what the bundle's images come from?
//
internal/bootstrap/trust.go
+317
View File
@@ -0,0 +1,317 @@
package bootstrap
import (
"context"
"encoding/json"
"fmt"
"net"
"os"
"strconv"
"strings"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92).
//
// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private
// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry
// it has not been told to trust, and the module that tells it — `docker`, which writes the
// artifact store's reach into the runtime's configuration — is applied long after genesis pushes
// the control plane's image. So genesis writes that trust itself, first, with the same writer the
// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and
// never restarted. It then asks the runtime whether it will speak to every name, and refuses with
// the runtime's answer before anything is pushed.
//
// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation
// could hold it, because the retire step re-applies the bundle and takes away a carried record the
// bundle does not name. The `docker` module, applied later, declares the artifact store's reach —
// the mesh name on the node's port — finds it already in the file it has no record of, and takes it
// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member
// does not change at the handover. The module's first apply usually still changes the file and
// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`,
// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name,
// stays as part of the machine's own configuration, as the hand-written trust it replaces would have.
//
// **This rests on an assumption about order.** The module takes the mesh name's member over only
// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the
// network step (16), and once the node is on the network the controller composes the reach as
// `<node>.<suffix>:<port>`.
// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:<port>`, as it is before the
// network — its record would exist without the mesh name, and the member would stay the machine's
// for good: undeclaring the module would never take it out. Installing the node's seat holders
// earlier in genesis must revisit this.
//
// **The suffix** is read from this process's environment, and the controller reads its own: a mesh
// with a custom `MESH_INTERNAL_SUFFIX` sets it for both.
// runtimeConfig is the container runtime's configuration, which the trust is written into. A
// variable so a test writes into a file of its own.
var runtimeConfig = "/etc/docker/daemon.json"
// runtimeUnit is the runtime's service, reloaded when the file changed.
const runtimeUnit = "docker.service"
// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable
// and default the controller composes `<node>.<suffix>` from (mesh-controller internal/overlay),
// which is the name the artifact store's reach — and so every push by the mesh's builder — uses.
const (
internalSuffixVar = "MESH_INTERNAL_SUFFIX"
defaultInternalSuffix = "internal"
)
// RegistryNames are the names the mesh will use for its registry on this node, each as host:port.
//
// The address the operator gave, and the node's mesh name: the controller's reach for the artifact
// store, which the builder pushes to without being told. Both on the port the registry module
// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A
// loopback name is left out: the runtime trusts loopback without being told, and the mesh never
// names it to another node.
func RegistryNames(o Options) []string {
port := strconv.Itoa(o.Ports.orDefaults().Registry)
var names []string
add := func(hostport string) {
host, _, err := net.SplitHostPort(hostport)
if err != nil || host == "" || isLoopback(host) {
return
}
for _, n := range names {
if n == hostport {
return
}
}
names = append(names, hostport)
}
if host, _, err := net.SplitHostPort(o.Registry); err == nil {
add(net.JoinHostPort(strings.Trim(host, "[]"), port))
}
if node := strings.TrimSpace(o.Node); node != "" {
add(net.JoinHostPort(node+"."+internalSuffix(), port))
}
return names
}
func internalSuffix() string {
if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" {
return strings.TrimPrefix(v, ".")
}
return defaultInternalSuffix
}
func isLoopback(host string) bool {
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(strings.Trim(host, "[]"))
return ip != nil && ip.IsLoopback()
}
// trustDeclaration is the declaration genesis applies to write the trust: the same two resources
// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it.
func trustDeclaration(names []string) (*declaration.Declaration, error) {
keys, err := json.Marshal(map[string][]string{"insecure-registries": names})
if err != nil {
return nil, err
}
raw, err := json.Marshal(map[string]any{
"declaration": 1,
"resources": []map[string]any{
{"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644",
"into": "json", "content": string(keys) + "\n"},
{"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit,
"state": "running", "reload-on": []string{"genesis.registry-trust"}},
},
})
if err != nil {
return nil, err
}
return declaration.ParseTrusted(raw)
}
// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here,
// then refuses unless the runtime says it will speak plain HTTP to each.
func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner,
say func(string)) ([]string, error) {
names := RegistryNames(o)
if len(names) == 0 {
say(" trusted only loopback names this registry, which the runtime trusts already")
return nil, nil
}
d, err := trustDeclaration(names)
if err != nil {
return names, err
}
// One apply at a time on this machine, as ApplyBundle does: a host already running here may be
// writing the same file.
unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") })
if err != nil {
return names, err
}
report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run,
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed)
unlock()
if err != nil {
return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+
"The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+
"runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err)
}
if report.Changed() {
say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig +
", the runtime reloaded")
} else {
say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so")
}
return names, runtimeWillSpeakTo(ctx, o, run, names)
}
// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out.
//
// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on
// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error.
func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error {
deadline := time.Now().Add(o.Wait)
var missing []string
var last error
for {
asking, cancel := context.WithTimeout(ctx, o.Timeout)
out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}")
cancel()
last = err
if err == nil {
missing, last = untrusted(out, names)
if last == nil && len(missing) == 0 {
return nil
}
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(answerEvery):
}
}
if last != nil {
return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last)
}
return fmt.Errorf(
"the container runtime does not trust %s after %s says it should, so it would refuse to push to "+
"it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+
"with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+
"Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit)
}
// untrusted reads the runtime's registry configuration and gives back the names it would speak
// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in
// a range the runtime trusts.
func untrusted(registryConfig string, names []string) ([]string, error) {
var c struct {
InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"`
IndexConfigs map[string]struct {
Secure bool `json:"Secure"`
} `json:"IndexConfigs"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil {
return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err)
}
var missing []string
for _, name := range names {
if index, ok := c.IndexConfigs[name]; ok && !index.Secure {
continue
}
host, _, _ := net.SplitHostPort(name)
if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) {
continue
}
missing = append(missing, name)
}
return missing, nil
}
func inAny(ip net.IP, cidrs []string) bool {
for _, cidr := range cidrs {
if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) {
return true
}
}
return false
}
// procRoot is where a process's command line is read from. A variable so a test gives its own.
var procRoot = "/proc"
// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known
// about it without writing anything, asked at step 1 so a machine that cannot take the trust is
// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92).
//
// - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to
// write over anything else (ADR 0102).
// - The runtime's service is known to systemd and active: the trust is taken by reloading it.
// - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both
// as a flag and in its file, so it would refuse the reload and keep distrusting the registry.
func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error {
names := RegistryNames(o)
if len(names) == 0 {
return nil
}
if raw, err := os.ReadFile(runtimeConfig); err == nil {
var object map[string]json.RawMessage
if len(strings.TrimSpace(string(raw))) > 0 {
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+
"(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+
"or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", "))
}
}
} else if !os.IsNotExist(err) {
return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err)
}
asking, cancel := context.WithTimeout(ctx, o.Timeout)
out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID")
cancel()
if err != nil {
return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+
"reloading it; nothing has been changed", runtimeUnit, err)
}
props := map[string]string{}
for _, line := range strings.Split(out, "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
props[k] = v
}
}
if props["LoadState"] != "loaded" || props["ActiveState"] != "active" {
return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+
"Genesis needs the runtime run by systemd as %s; nothing has been changed",
runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]),
strings.Join(names, ", "), runtimeUnit)
}
if pid := props["MainPID"]; pid != "" && pid != "0" {
if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil {
for _, arg := range strings.Split(string(cmdline), "\x00") {
if strings.HasPrefix(arg, "--insecure-registry") {
return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+
"given both as a flag and in %s — so it would refuse the reload that makes it trust "+
"%s. Move the flag's registries into the file and restart the runtime once; nothing "+
"has been changed", arg, runtimeConfig, strings.Join(names, ", "))
}
}
}
}
say(" registry trust " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded")
return nil
}
func orUnknown(s string) string {
if s == "" {
return "unknown"
}
return s
}
internal/bootstrap/trust_test.go
+347
View File
@@ -0,0 +1,347 @@
package bootstrap
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
// told the container runtime to trust that registry, so the push failed with "server gave HTTP
// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
// and the trust the mesh wrote later named the registry's default port rather than the node's.
// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
type aRuntime struct {
path string
trusted []string
commands []string
// ignoresReload is a runtime started with options of its own, which a reload does not change.
ignoresReload bool
}
func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
r.commands = append(r.commands, line)
switch {
case name == "docker" && len(args) > 0 && args[0] == "info":
index := map[string]map[string]any{"docker.io": {"Secure": true}}
for _, n := range r.trusted {
index[n] = map[string]any{"Name": n, "Secure": false}
}
out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
return string(out), nil
case name == "systemctl" && strings.Contains(line, "reload"):
if !r.ignoresReload {
r.trusted = r.trusted[:0]
raw, _ := os.ReadFile(r.path)
var c struct {
Insecure []string `json:"insecure-registries"`
}
_ = json.Unmarshal(raw, &c)
r.trusted = append(r.trusted, c.Insecure...)
}
case strings.Contains(line, "is-enabled"):
return "enabled", nil
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
}
return "", nil
}
func (r *aRuntime) did(word string) int {
n := 0
for _, c := range r.commands {
if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
n++
}
}
return n
}
func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
t.Helper()
restore := apply.ForTests(t.TempDir(), t.TempDir())
t.Cleanup(restore)
dir := t.TempDir()
was := runtimeConfig
runtimeConfig = filepath.Join(dir, "daemon.json")
t.Cleanup(func() { runtimeConfig = was })
if existing != "" {
if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
t.Fatal(err)
}
}
t.Setenv(internalSuffixVar, "")
sys, err := system.For("arch")
if err != nil {
t.Fatal(err)
}
o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
return o, &aRuntime{path: runtimeConfig}, sys
}
func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
t.Helper()
raw, err := os.ReadFile(runtimeConfig)
if err != nil {
t.Fatal(err)
}
var o map[string]json.RawMessage
if err := json.Unmarshal(raw, &o); err != nil {
t.Fatal(err)
}
var list []string
_ = json.Unmarshal(o["insecure-registries"], &list)
return fmt.Sprint(list), o
}
func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)
names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
if err != nil {
t.Fatal(err)
}
// The address given and the mesh name the builder pushes by, both on the node's port.
if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
t.Errorf("trusted by %v", names)
}
list, object := insecureIn(t)
if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
t.Errorf("the runtime's list is %s", list)
}
if string(object["data-root"]) != `"/srv/docker"` {
t.Errorf("the machine's own key was not kept: %s", object["data-root"])
}
if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
}
// Re-run: nothing changes, nothing is reloaded.
runtime.commands = nil
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
t.Fatal(err)
}
if runtime.did("reload") != 0 {
t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
}
}
// The port is the node's setting for the registry, not the module's default, whatever the operator
// typed the address with — and loopback, which the runtime trusts already, is not written.
func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
t.Setenv(internalSuffixVar, "")
if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
t.Errorf("names %s", got)
}
o.Ports = FoundationPorts{}
o.Registry = "192.0.2.10:5000"
t.Setenv(internalSuffixVar, ".mesh.example")
if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
t.Errorf("names %s", got)
}
}
// A runtime that still refuses after the write is refused here, with the names, before any push.
func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
o, runtime, sys := trustSetup(t, "")
runtime.ignoresReload = true
_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
if err == nil {
t.Fatal("a runtime that does not trust the registry was let through to the push")
}
if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
!strings.Contains(err.Error(), "Nothing has been pushed") {
t.Errorf("the refusal does not say what: %v", err)
}
}
// The handover: the `docker` module, applied later, writes the reach the controller composes for
// the artifact store — the mesh name and the node's port. It finds it there: the file does not
// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
// owns the trust it writes and taking the module away takes that member out.
func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
t.Fatal(err)
}
before, _ := os.ReadFile(runtimeConfig)
module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
"content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
{"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
"reload-on":["docker.daemon"]}]}`, runtimeConfig)))
if err != nil {
t.Fatal(err)
}
runtime.commands = nil
_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
runtime.run, nil, nil)
if err != nil {
t.Fatal(err)
}
after, _ := os.ReadFile(runtimeConfig)
if string(before) != string(after) {
t.Errorf("the handover changed the file:\n%s\n%s", before, after)
}
if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
t.Errorf("the handover touched the runtime: %v", runtime.commands)
}
if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
}
}
func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
[]string{"10.42.0.1:5000", "anchor.internal:5000"})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(missing) != "[anchor.internal:5000]" {
t.Errorf("missing %v", missing)
}
}
// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves
// as dockerd does: a push to a registry it has not been told to trust is refused with the error the
// first genesis met. The trust must be written and confirmed before anything is tagged or pushed;
// moving the trust after the push, or dropping it, fails this test.
func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`)
o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest)
o.Registry = "192.0.2.10:5100"
pushed := false
mesh := aMeshThatAgrees(nil)
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "systemctl" || (name == "docker" && args[0] == "info") {
return runtime.run(ctx, name, args...)
}
runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " "))
if name == "docker" {
switch args[0] {
case "tag":
return "", nil
case "push":
trusted := false
for _, n := range runtime.trusted {
trusted = trusted || strings.HasPrefix(args[1], n+"/")
}
if !trusted {
return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry)
}
pushed = true
return "", nil
case "image":
return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
}
}
return mesh(name, args)
}
d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) {
if strings.HasSuffix(url, "/v2/") || pushed {
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
}
return http.StatusNotFound, "", nil
}}
var result Result
published, err := RegistryAndPublish(context.Background(), o, d, sys,
controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {})
if err != nil {
t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands)
}
if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") {
t.Errorf("published as %q", published.Reference)
}
firstPush, reloaded, confirmed := -1, -1, -1
for i, c := range runtime.commands {
switch {
case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")):
firstPush = i
case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"):
reloaded = i
case strings.HasPrefix(c, "docker info"):
confirmed = i
}
}
if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) {
t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v",
reloaded, confirmed, firstPush, runtime.commands)
}
if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" {
t.Errorf("the result says it trusted %v", result.RegistryTrusted)
}
}
// Preflight asks, without writing, what would stop the trust from taking at step 7.
func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) {
proc := t.TempDir()
was := procRoot
procRoot = proc
t.Cleanup(func() { procRoot = was })
if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil {
t.Fatal(err)
}
unit := func(load, active string) Runner {
return func(_ context.Context, name string, args ...string) (string, error) {
return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil
}
}
cmdline := func(args ...string) {
if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil {
t.Fatal(err)
}
}
cmdline("/usr/bin/dockerd", "-H", "fd://")
for _, c := range []struct {
name, file string
run Runner
flag bool
refused string
}{
{"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""},
{"no file yet", "", unit("loaded", "active"), false, ""},
{"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"},
{"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"},
{"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"},
} {
t.Run(c.name, func(t *testing.T) {
o, _, _ := trustSetup(t, c.file)
if c.flag {
cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000")
defer cmdline("/usr/bin/dockerd", "-H", "fd://")
}
before, _ := os.ReadFile(runtimeConfig)
err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {})
switch {
case c.refused == "" && err != nil:
t.Errorf("refused: %v", err)
case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)):
t.Errorf("wanted a refusal naming %q, got %v", c.refused, err)
}
if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) {
t.Error("preflight wrote the runtime's file")
}
})
}
}