mesh-catalog/modules/nats/cmd/mesh-nats-snapshot/image_test.go

//go:build integration

package main

// The declared dump and the mounted entrypoint, exactly as the manifest declares them, against the upstream image
// the manifest pins: the mesh's snapshot program and entrypoint mounted at /opt (novox/hq ADR 0480 clauses 1, 2, 5).
//
//	go test -tags integration -run 'TestTheDeclaredDump|TestTheEntrypoint|TestTheStoreSurvives' .   (through mesh-controller.try)
//
// The image is pulled by the manifest's reference, so the server these run is the one the bus runs.

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"nats-tools/internal/busarchive"
)

// manifestOf is the nats module's own definition, beside this program.
func manifestOf(t *testing.T) map[string]any {
	t.Helper()
	raw, err := os.ReadFile("../../module.json")
	if err != nil {
		t.Fatal(err)
	}
	var m map[string]any
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatal(err)
	}
	return m
}

func declaredDump(t *testing.T, m map[string]any) string {
	t.Helper()
	for _, item := range m["data"].(map[string]any)["own"].([]any) {
		it := item.(map[string]any)
		if it["id"] != "jetstream" {
			continue
		}
		backup, _ := it["backup"].(map[string]any)
		if backup == nil || backup["into"] != "snapshots" {
			t.Fatalf("the bus's streams are not protected by a dump into its snapshots: %v", it["backup"])
		}
		return backup["dump"].(string)
	}
	t.Fatal("the nats module declares no jetstream data item")
	return ""
}

// serverConf is the module's own server settings, from the template its resource names.
func serverConf(t *testing.T, m map[string]any) string {
	t.Helper()
	for _, r := range m["resources"].([]any) {
		res := r.(map[string]any)
		if res["id"] == "server-conf" {
			return templateText(t, res)
		}
	}
	t.Fatal("the nats module declares no server configuration")
	return ""
}

// busImage is the upstream image the manifest pins, as a reference docker pulls (name:tag@digest).
func busImage(t *testing.T, m map[string]any) string {
	t.Helper()
	for _, a := range m["build"].(map[string]any)["artifacts"].([]any) {
		art := a.(map[string]any)
		if art["name"] == "server" {
			if art["kind"] != "upstream" {
				t.Fatalf("the server artifact is a %v, not an upstream image", art["kind"])
			}
			return art["from"].(string)
		}
	}
	t.Fatal("the nats module declares no server artifact")
	return ""
}

// programs builds the snapshot program for alpine (static, as the mesh's build does) and lays out the two
// directories the manifest mounts: the program at /opt/mesh and the entrypoint at /opt/mesh-entrypoint.
func programs(t *testing.T) (program, entry string) {
	t.Helper()
	program, entry = t.TempDir(), t.TempDir()
	build := exec.Command("go", "build", "-trimpath", "-o", filepath.Join(program, "mesh-nats-snapshot"), ".")
	build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux")
	if out, err := build.CombinedOutput(); err != nil {
		t.Fatalf("building the program: %v\n%s", err, out)
	}
	script, err := os.ReadFile(filepath.Join(moduleDir, "program", "entrypoint.sh"))
	if err != nil {
		t.Fatal(err)
	}
	// Mode 0644: the manifest runs the script through sh, so no execute bit is relied on.
	if err := os.WriteFile(filepath.Join(entry, "entrypoint.sh"), script, 0o644); err != nil {
		t.Fatal(err)
	}
	return program, entry
}

// runBus starts the upstream image the way the manifest does: its volumes, its args, the module's configuration
// and the given user list. The poll is a second so a test need not wait out the production five.
func runBus(t *testing.T, m map[string]any, name, accounts string) (port, pin, conf string) {
	t.Helper()
	if out, _ := exec.Command("docker", "ps", "-a", "-q", "--filter", "name=^"+name+"$").Output(); len(strings.TrimSpace(string(out))) > 0 {
		t.Skipf("a container called %s already runs here; this test will not touch it", name)
	}
	conf, tlsDir, data := t.TempDir(), t.TempDir(), t.TempDir()
	pin = selfSigned(t, tlsDir)
	_ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(serverConf(t, m)), 0o644)
	_ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(accounts), 0o644)
	program, entry := programs(t)
	docker(t, "run", "-d", "--rm", "--name", name, "--user", owner(), "-p", "127.0.0.1::4222",
		"-e", "MESH_NATS_CONF_POLL_SECONDS=1",
		"-v", data+":/data", "-v", conf+":/etc/nats:ro", "-v", tlsDir+":/tls:ro",
		"-v", program+":/opt/mesh:ro", "-v", entry+":/opt/mesh-entrypoint:ro",
		busImage(t, m), "sh", "/opt/mesh-entrypoint/entrypoint.sh")
	t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() })
	_, port, _ = strings.Cut(docker(t, "port", name, "4222/tcp"), ":")
	return port, pin, conf
}

// The dump is the one the manifest declares, run by `sh -c` as the holder runs it, with the module's
// directories filled in; the server is the module's image with the module's own configuration and a
// user list holding the bus's own module with the controller's grants for it.
func TestTheDeclaredDumpRunsInTheBussImage(t *testing.T) {
	needDocker(t)
	const name = "mesh-broker-nats" // the container the dump names
	m := manifestOf(t)
	dump := declaredDump(t, m)

	state, snapshots := t.TempDir(), t.TempDir()
	accounts := liveBusConf()
	accounts = accounts[strings.Index(accounts, "accounts {"):]
	port, pin, _ := runBus(t, m, name, accounts)

	admin := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
	js, _ := admin.JetStream()
	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_hand-acts"})
	if err != nil {
		t.Fatal(err)
	}
	for _, k := range []string{"a", "b", "c"} {
		if _, err := kv.Put(k, []byte("act "+k)); err != nil {
			t.Fatal(err)
		}
	}

	run := func(cred Credential) (string, error) {
		raw, _ := json.Marshal(cred)
		if err := os.WriteFile(filepath.Join(state, "broker"), raw, 0o600); err != nil {
			t.Fatal(err)
		}
		command := strings.NewReplacer("${dir:mesh-state}", state, "${dir:snapshots}", snapshots).Replace(dump)
		out, err := exec.Command("sh", "-c", command).CombinedOutput()
		return string(out), err
	}

	began := time.Now()
	out, err := run(Credential{URL: "nats://bus.example:4222", Fingerprint: pin, User: snapshotUser, Password: "snap"})
	if err != nil {
		t.Fatalf("the declared dump failed: %v\n%s", err, out)
	}
	t.Logf("the declared dump took %s:\n%s", time.Since(began).Round(time.Millisecond), out)
	tarPath := filepath.Join(snapshots, "bus.tar")
	f, err := os.Open(tarPath)
	if err != nil {
		t.Fatalf("the dump left no bus.tar: %v", err)
	}
	a, err := busarchive.ReadArchive(f, "")
	f.Close()
	if err != nil {
		t.Fatal(err)
	}
	defer a.Close()
	if err := a.Verify(); err != nil {
		t.Fatal(err)
	}
	if len(a.Manifest.Streams) != 1 || a.Manifest.Streams[0].Name != "KV_mesh-controller_hand-acts" || a.Manifest.Streams[0].Messages != 3 {
		t.Fatalf("the snapshot holds %+v", a.Manifest.Streams)
	}
	before, _ := os.ReadFile(tarPath)

	// A night that cannot snapshot fails, says why, and leaves last night's snapshot as it was.
	out, err = run(Credential{Fingerprint: pin, User: snapshotUser, Password: "wrong"})
	if err == nil {
		t.Fatalf("a dump the bus refused succeeded:\n%s", out)
	}
	if !strings.Contains(out, "Authorization Violation") && !strings.Contains(strings.ToLower(out), "authorization") {
		t.Errorf("the failure does not say the bus refused the credential: %s", out)
	}
	after, _ := os.ReadFile(tarPath)
	if string(after) != string(before) {
		t.Error("a failed night replaced the last good snapshot")
	}
	if _, err := os.Stat(tarPath + ".partial"); !os.IsNotExist(err) {
		t.Error("a failed night left its partial file behind")
	}
}

// **A change of the user list reloads the server and keeps its connections** (ADR 0480 clause 5, ADR 0236 §37).
// The mounted entrypoint watches accounts.conf and signals the server; a client connected meanwhile is not
// dropped, and a user added by the change can then connect.
func TestTheEntrypointReloadsTheServerWithoutDroppingAConnection(t *testing.T) {
	needDocker(t)
	m := manifestOf(t)
	accounts := plainConf()
	accounts = accounts[strings.Index(accounts, "accounts {"):]
	port, pin, conf := runBus(t, m, fmt.Sprintf("mesh-reload-test-%d", os.Getpid()), accounts)

	admin := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
	if err := admin.Flush(); err != nil {
		t.Fatal(err)
	}

	changed := strings.Replace(accounts, `{ user: "admin", password: "admin" }`,
		`{ user: "admin", password: "admin" }, { user: "late", password: "late" }`, 1)
	if changed == accounts {
		t.Fatal("the test's user list did not change")
	}
	// Written as the mesh writes it: replaced, not edited in place, so the watcher sees whole content.
	tmp := filepath.Join(conf, "accounts.conf.new")
	if err := os.WriteFile(tmp, []byte(changed), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.Rename(tmp, filepath.Join(conf, "accounts.conf")); err != nil {
		t.Fatal(err)
	}

	// dial retries until the server takes the user: the reload happens within the poll interval.
	late := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "late", Password: "late"}, nil)
	_ = late
	if !admin.IsConnected() {
		t.Fatal("the connection open before the reload was dropped by it")
	}
	if err := admin.Flush(); err != nil {
		t.Fatalf("the connection open before the reload no longer works: %v", err)
	}
	if n := admin.Stats().Reconnects; n != 0 {
		t.Fatalf("the connection open before the reload reconnected %d times", n)
	}
}

// **The bus can go back** (ADR 0480 clause 6): the new declaration and then the old one run on the same store, both
// well, which is what makes `reversible` true for the bus upgrade. The server binary (the upstream image's digest),
// the data directory and the configuration are the same in both; only where the entrypoint and the program come
// from differs. The old declaration is run as the image it replaced ran it: the same script as the image's own
// entrypoint, the program on the image's path. Then the new one again, so a way there, back and there again is
// shown, each time with the stream's messages as the one before left them.
func TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain(t *testing.T) {
	needDocker(t)
	m := manifestOf(t)
	accounts := liveBusConf()
	accounts = accounts[strings.Index(accounts, "accounts {"):]
	conf, tlsDir, data := t.TempDir(), t.TempDir(), t.TempDir()
	pin := selfSigned(t, tlsDir)
	_ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(serverConf(t, m)), 0o644)
	_ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(accounts), 0o644)
	program, entry := programs(t)
	script := filepath.Join(entry, "entrypoint.sh")
	if err := os.Chmod(script, 0o755); err != nil {
		t.Fatal(err)
	}
	common := []string{"-d", "--rm", "--user", owner(), "-p", "127.0.0.1::4222", "-e", "MESH_NATS_CONF_POLL_SECONDS=1",
		"-v", data + ":/data", "-v", conf + ":/etc/nats:ro", "-v", tlsDir + ":/tls:ro"}
	declarations := map[string][]string{
		"new": {"-v", program + ":/opt/mesh:ro", "-v", entry + ":/opt/mesh-entrypoint:ro", busImage(t, m), "sh", "/opt/mesh-entrypoint/entrypoint.sh"},
		"old": {"-v", script + ":/usr/local/bin/mesh-nats-entrypoint:ro",
			"-v", filepath.Join(program, "mesh-nats-snapshot") + ":/usr/local/bin/mesh-nats-snapshot:ro",
			"--entrypoint", "/usr/local/bin/mesh-nats-entrypoint", busImage(t, m)},
	}
	var kv nats.KeyValue
	puts := 0
	step := func(which string) {
		name := fmt.Sprintf("mesh-rehearsal-%s-%d", which, os.Getpid())
		docker(t, append(append([]string{"run", "--name", name}, common...), declarations[which]...)...)
		t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() })
		_, port, _ := strings.Cut(docker(t, "port", name, "4222/tcp"), ":")
		c := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
		js, _ := c.JetStream()
		var err error
		if puts == 0 {
			kv, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-rehearsal"})
		} else {
			kv, err = js.KeyValue("mesh-rehearsal")
		}
		if err != nil {
			t.Fatalf("%s declaration: the store's bucket: %v", which, err)
		}
		for i := 1; i <= puts; i++ {
			e, err := kv.Get(fmt.Sprintf("k%d", i))
			if err != nil || string(e.Value()) != fmt.Sprintf("v%d", i) {
				t.Fatalf("%s declaration: key k%d reads %v, %v", which, i, e, err)
			}
		}
		puts++
		if _, err := kv.Put(fmt.Sprintf("k%d", puts), []byte(fmt.Sprintf("v%d", puts))); err != nil {
			t.Fatal(err)
		}
		c.Close()
		docker(t, "stop", "-t", "10", name)
	}
	step("new")
	step("old")
	step("new")
}