mesh-lab/replays/resolver_test.go

package replays

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"text/template"
	"time"
)

// **R262 — every machine's name resolves under musl and under glibc** (novox/hq issue 262; to-be 45 §9,
// "the resolver module's tests run under musl and glibc").
//
// After the mesh moved to one resolver, an Alpine container crash-looped on `getaddrinfo ENOTFOUND` for
// a machine's mesh name that `getent hosts` found: the resolver answered the name only through a
// wildcard, so asked for its IPv6 address it said there is no such name, and musl — which asks for both
// and takes NXDOMAIN as final — gave up; glibc did not. The replay renders the machine list the
// catalogue's resolver module reads (its `node-zones` fact, from the manifest at MESH_REPLAY_CATALOGUE),
// runs the resolver with it, and asks for each machine by `getaddrinfo` from a program under each C
// library, on the runtime's default bridge as the incident's container was. Needs a container runtime
// (its socket); its images come from the public registry.
var libcs = map[string]string{"musl": "python:3.13-alpine", "glibc": "python:3.13-slim"}

// resolverImage is where the resolver runs: the distribution's own dnsmasq, installed as the module
// installs it on a machine.
const resolverImage = "alpine:3.20"

func TestReplay262EveryMachineResolvesUnderMuslAndGlibc(t *testing.T) {
	catalogue := os.Getenv("MESH_REPLAY_CATALOGUE")
	if catalogue == "" {
		catalogue = filepath.Join("..", "..", "mesh-catalog")
	}
	raw, err := os.ReadFile(filepath.Join(catalogue, "modules", "dnsmasq", "module.json"))
	if err != nil {
		t.Skipf("no catalogue to read the resolver's configuration from (%v); MESH_REPLAY_CATALOGUE names one", err)
	}
	docker, ok := DockerFromEnv()
	if !ok {
		t.Skip("no container runtime: the resolver and the two C libraries run in containers")
	}
	var manifest struct {
		Facts map[string]struct {
			Template string `json:"template"`
		} `json:"facts"`
	}
	if err := json.Unmarshal(raw, &manifest); err != nil {
		t.Fatal(err)
	}
	zones, ok := manifest.Facts["node-zones"]
	if !ok {
		t.Fatal("the resolver module renders no machine list (`facts.node-zones`)")
	}
	type entry struct{ Name, FQDN, Address, Account string }
	machines := []entry{{Name: "anchor", FQDN: "anchor.internal", Address: "10.88.0.1"},
		{Name: "homeserver", FQDN: "homeserver.internal", Address: "10.88.0.2"}}
	var rendered bytes.Buffer
	tpl, err := template.New("node-zones").Parse(zones.Template)
	if err == nil {
		err = tpl.Execute(&rendered, map[string]any{"Node": "anchor", "Suffix": "internal", "Names": machines,
			"Machines": machines, "Zones": nil, "Holders": map[string][]entry{}})
	}
	if err != nil {
		t.Fatalf("the machine list does not render: %v", err)
	}

	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
	defer cancel()
	for _, image := range append([]string{resolverImage}, libcs["musl"], libcs["glibc"]) {
		if err := docker.Pull(ctx, image); err != nil {
			t.Fatalf("cannot fetch %s: %v", image, err)
		}
	}
	// The resolver: nothing but the machine list, answering on its address on the bridge.
	conf := base64.StdEncoding.EncodeToString(rendered.Bytes())
	script := fmt.Sprintf(`apk add -q --no-cache dnsmasq >/dev/null && echo %s | base64 -d > /etc/nodes.conf && `+
		`exec dnsmasq -k --no-resolv --no-hosts --log-queries --log-facility=- --conf-file=/etc/nodes.conf`, conf)
	resolver, address, err := docker.Start(ctx, Run{Image: resolverImage, Cmd: []string{"sh", "-c", script}, Network: "bridge"})
	if err != nil {
		t.Fatal(err)
	}
	defer docker.Remove(context.Background(), resolver)
	if address == "" {
		t.Fatal("the resolver has no address on the bridge")
	}

	// Each machine, asked by getaddrinfo as a program does — both families, as musl asks — a few times
	// while the resolver comes up.
	ask := fmt.Sprintf(`
import socket, sys, time
names = %q.split()
for attempt in range(30):
    failed = []
    for n in names:
        try:
            socket.getaddrinfo(n, 80)
        except socket.gaierror as e:
            failed.append("%%s: %%s" %% (n, e))
    if not failed:
        print("resolved", " ".join(names)); sys.exit(0)
    time.sleep(1)
print("\n".join(failed)); sys.exit(1)
`, machines[0].FQDN+" "+machines[1].FQDN)
	for libc, image := range libcs {
		id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"python3", "-c", ask}, Network: "bridge",
			DNS: []string{address}})
		if err != nil {
			t.Fatal(err)
		}
		code, said, err := docker.Wait(ctx, id)
		docker.Remove(context.Background(), id)
		if err != nil {
			t.Fatal(err)
		}
		if code != 0 {
			t.Errorf("under %s (%s) a machine's mesh name does not resolve through the catalogue's resolver — "+
				"issue 262:\n%s\nthe resolver said:\n%s", libc, image, strings.TrimSpace(said), tail(docker.Logs(ctx, resolver), 12))
		}
	}
}

func tail(s string, n int) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, "\n")
}