mesh-controller/internal/catalogue/resolver_manifests_test.go

package catalogue

import (
	"encoding/json"
	"strings"
	"testing"
)

// The catalogue's resolver modules as they are, parsed by the real parser and composed as a
// machine would receive them (hal dnsmasq-app conversion, novox/hq 08-connectivity).
//
// The predecessor's resolver answered every name on a machine: the mesh's own itself, the rest
// forwarded to two fixed upstreams, with the machine's resolv.conf naming it alone and the
// container runtime pointed at its private-network address. These hold the mesh's modules to the
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
// its upstreams, or take an address systemd-resolved holds.

// resolverShelf is the three resolver modules beside something that answers `mesh-addressing`.
// The networking module that really does is composed in the controller and cannot be imported
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
func resolverShelf(t *testing.T) map[string]Manifest {
	t.Helper()
	shelf := map[string]Manifest{
		"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
	}
	for _, name := range []string{"dnsmasq", "resolv-conf", "resolved-split-dns"} {
		shelf[name] = catalogueManifest(t, name)
	}
	return shelf
}

// twoMachines is what the control plane hands a rendering: internal names and their addresses.
var twoMachines = map[string]string{"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2"}

// Its configuration forwards to the upstreams the predecessor's module shipped, and gets them from
// nowhere else: `no-resolv` is what makes the documented loop — the resolver finding its own
// address in resolv.conf and becoming its own upstream — impossible.
func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T) {
	m := catalogueManifest(t, "dnsmasq")
	var config string
	for _, r := range m.Resources {
		if r["id"] == "config" {
			config, _ = r["content"].(string)
		}
	}
	if config == "" {
		t.Fatal("the resolver has no configuration file")
	}
	for _, want := range []string{
		"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
		// The private address and loopback, never a LAN's (novox/hq ADR 0194): a device that is not a
		// member cannot reach what the mesh's names point at.
		"\nlisten-address=127.0.0.1\n", "\nlisten-address=${machine:address}\n", "\nbind-dynamic\n",
		// No hosts file and no operator's files: the mesh's resolver answers every node (ADR 0199).
		"\nno-hosts\n",
		"\nconf-file=" + m.Facts["zones"].Path + "\n",
		"\ndomain-needed\n", "\nbogus-priv\n",
		"\nconf-file=" + m.Facts["node-zones"].Path + "\n",
	} {
		if !strings.Contains(config, want) {
			t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
		}
	}
	// By address and never by interface: dnsmasq admits a query by the interface it arrives on
	// when told one, and a container's query to the private address arrives on the runtime's
	// bridge — `interface=mesh0` dropped every such query, silently (novox/hq issue 110).
	for _, line := range strings.Split(config, "\n") {
		if strings.HasPrefix(line, "interface=") {
			t.Errorf("the resolver answers by interface, so a container's query on a bridge is dropped: %s", line)
		}
	}
	// Not .53 or .54, which systemd-resolved holds; and not .55 any more, which was a convention
	// beside the one every machine already followed — the predecessor's resolv.conf says .1.
	for _, taken := range []string{"127.0.0.53", "127.0.0.54", "127.0.0.55"} {
		if strings.Contains(config, "listen-address="+taken) {
			t.Errorf("the resolver listens on %s", taken)
		}
	}
	// Never a directory or a file the operator keeps: a line written for one machine's programs would
	// become an answer for every node (ADR 0199).
	for _, never := range []string{"conf-dir=", "addn-hosts=", "listen-address=${setting:"} {
		if strings.Contains(config, never) {
			t.Errorf("the mesh's resolver still reads or listens on %q", never)
		}
	}
	// And the file that decides what the machine asks names the mesh's resolver first, by address,
	// and a public one second, asked only when the first is silent (ADR 0196).
	var resolv string
	for _, r := range catalogueManifest(t, "resolv-conf").Resources {
		if r["path"] == "/etc/resolv.conf" {
			resolv, _ = r["content"].(string)
		}
	}
	var nameservers []string
	for _, line := range strings.Split(resolv, "\n") {
		if strings.HasPrefix(line, "nameserver ") {
			nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
		}
	}
	if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
		t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
	}
	if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
		t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
	}
	// The split-DNS alternative points at the same resolver, or a machine that keeps
	// systemd-resolved in charge would route the mesh's suffix to nothing.
	for _, r := range catalogueManifest(t, "resolved-split-dns").Resources {
		if content, _ := r["content"].(string); content != "" && !strings.Contains(content, "DNS=${bound:wildcard-resolution:address}\n") {
			t.Errorf("resolved-split-dns does not point at the resolver's address:\n%s", content)
		}
	}
}

// The resolver and what points the machine at it compose on one machine, and what arrives is the
// mesh's account of every machine as a wildcard, the suffix kept local, the daemon restarting on
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
// its own but kept running across a restart (ADR 0196).
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
	got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf"},
		Node{Name: "anchor", At: "anchor.internal"}, World{})
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(strings.Join(named(got), " "), "net") {
		t.Fatalf("the resolver's data is the mesh's addresses, and nothing answering them was taken: %v", named(got))
	}
	out, err := got.Declaration(Rendering{
		// Names is every name the mesh serves; Machines is the subset that is a node (novox/hq
		// issue 111) — the resolver's zones read only the second, and in this scenario the two
		// happen to be the same map, since nothing routed is part of it.
		Names: twoMachines, Machines: twoMachines, Suffix: "internal",
		Zones:    []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
		Needed:   map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
		Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
	})
	if err != nil {
		t.Fatal(err)
	}

	ids := byID(out)
	zones := ids["dnsmasq.fact-node-zones"]
	if zones == nil || zones["path"] != "/etc/mesh-resolver/nodes.conf" {
		t.Fatalf("the resolver was not given the machines where its configuration reads them: %v", zones)
	}
	content, _ := zones["content"].(string)
	for _, want := range []string{
		"local=/internal/", "address=/anchor.internal/10.42.0.1", "address=/laptop.internal/10.42.0.2",
	} {
		if !strings.Contains(content, want) {
			t.Errorf("the machines file lacks %q:\n%s", want, content)
		}
	}

	service := ids["dnsmasq.service"]
	if service == nil {
		t.Fatal("no resolver service composed")
	}
	reflects := map[string]bool{}
	for _, id := range service["restart-on"].([]any) {
		reflects[id.(string)] = true
	}
	if !reflects["dnsmasq.config"] || !reflects["dnsmasq.fact-node-zones"] || !reflects["dnsmasq.fact-zones"] {
		t.Errorf("the daemon does not restart on its configuration, the machines and the zones: %v", service["restart-on"])
	}
	if z, _ := ids["dnsmasq.fact-zones"]["content"].(string); !strings.Contains(z, "server=/incus/10.42.0.2#5353\n") {
		t.Errorf("the resolver was not told to forward the zone to its answerer:\n%s", z)
	}

	// The runtime's own file, written into (novox/hq ADR 0102) with one key, by what decides how the
	// machine resolves: a restart keeps every container running. No `dns` — a container copies its
	// machine's resolvers (ADR 0196), and the mesh's resolver is not written into the runtime twice.
	if ids["dnsmasq.runtime-dns"] != nil {
		t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
	}
	runtime := ids["resolv-conf.runtime-config"]
	if runtime == nil || runtime["path"] != "/etc/docker/daemon.json" || runtime["into"] != "json" {
		t.Fatalf("live-restore is not written into the runtime's file: %v", runtime)
	}
	var keys map[string]any
	if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil {
		t.Fatalf("the runtime's keys are not JSON: %v", err)
	}
	if len(keys) != 1 || keys["live-restore"] != true {
		t.Errorf("the runtime is given %v; live-restore and nothing else", keys)
	}
	// The runtime is reloaded when that file changes, and never restarted: a restart stops every
	// container on the machine (ADR 0102), and a reload is what turns live-restore on.
	var reloaded bool
	for _, r := range out {
		if r["type"] != "service" || r["unit"] != "docker.service" {
			continue
		}
		if _, restarts := r["restart-on"]; restarts {
			t.Errorf("the runtime is ordered restarted, which stops every container (ADR 0102): %v", r)
		}
		for _, on := range asStrings(r["reload-on"]) {
			if on == "resolv-conf.runtime-config" {
				reloaded = true
			}
		}
	}
	if !reloaded {
		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
	}

	resolv := ids["resolv-conf.resolv"]
	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
		t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
	}
}

// Two modules deciding what a machine asks are refused on one machine, as before — the claim
// exists so they never take turns overwriting each other.
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
	_, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "resolved-split-dns"},
		Node{Name: "anchor", At: "anchor.internal"}, World{})
	if err == nil {
		t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
	}
	if !strings.Contains(err.Error(), "node-resolver-config") {
		t.Fatalf("the refusal does not say what was claimed: %v", err)
	}
}

// A machine that is not on the private network has no address for the runtime to be pointed at.
// Refused where the module and the machine are both named, rather than a placeholder written into
// the runtime's file and read as an address.
func TestTheResolverOnAMachineOffTheNetworkIsRefused(t *testing.T) {
	got, err := Resolve(resolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor"}, World{})
	if err != nil {
		t.Fatal(err)
	}
	// Left out of the declaration and said, rather than composed listening nowhere: a module that
	// cannot compose on a machine is kept as it is there, with the reason (hq ADR 0163).
	composed, err := got.Compose(Rendering{Names: twoMachines, Suffix: "internal",
		Needed:   map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
		Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}}})
	if err == nil && !strings.Contains(composed.LeftOut["dnsmasq"], "${machine:address}") {
		t.Fatalf("a machine off the network was composed a resolver, or left out for another reason: %v",
			composed.LeftOut)
	}
	if err != nil && !strings.Contains(err.Error(), "${machine:address}") {
		t.Fatalf("a machine off the network was refused for another reason: %v", err)
	}
}