mesh-catalog/modules/nats/cmd/nats-tools/rehearse_integration_test.go

//go:build integration

package main

// The rehearsal against real containers: a source server filled and snapshotted by the real program, then the
// rehearsal run on that bus.tar with the manifest's pinned image, the real program and the real entrypoint. Run through
// mesh-controller.try (the merge check runs it with -tags integration); never by hand on a workstation.

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
)

func sh(t *testing.T, name string, args ...string) string {
	t.Helper()
	out, err := exec.Command(name, args...).CombinedOutput()
	if err != nil {
		t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out)
	}
	return strings.TrimSpace(string(out))
}

func TestARehearsalRunsTheNewAndTheOldDeclarationOnACopyOfARealSnapshot(t *testing.T) {
	if _, err := exec.LookPath("docker"); err != nil {
		t.Fatalf("this test needs docker on the machine that runs it: %v", err)
	}
	raw, err := os.ReadFile("../../module.json")
	if err != nil {
		t.Fatal(err)
	}
	var m struct {
		Build struct {
			Artifacts []struct{ Name, From string } `json:"artifacts"`
		} `json:"build"`
	}
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatal(err)
	}
	image := ""
	for _, a := range m.Build.Artifacts {
		if a.Name == "server" {
			image = a.From
		}
	}
	work := t.TempDir()
	program, entry, snaps := filepath.Join(work, "program"), filepath.Join(work, "entry"), filepath.Join(work, "snaps")
	for _, d := range []string{program, entry, snaps, filepath.Join(work, "srcdata")} {
		if err := os.MkdirAll(d, 0o777); err != nil {
			t.Fatal(err)
		}
		_ = os.Chmod(d, 0o777)
	}
	build := exec.Command("go", "build", "-trimpath", "-o", filepath.Join(program, "mesh-nats-snapshot"), "../mesh-nats-snapshot")
	build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux")
	if out, err := build.CombinedOutput(); err != nil {
		t.Fatalf("building the snapshot program: %v\n%s", err, out)
	}
	script, err := os.ReadFile("../../program/entrypoint.sh")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(entry, "entrypoint.sh"), script, 0o644); err != nil {
		t.Fatal(err)
	}

	// The source: a plain server with the one account, filled, and snapshotted by the real program.
	conf := filepath.Join(work, "srcconf")
	if err := os.MkdirAll(conf, 0o755); err != nil {
		t.Fatal(err)
	}
	_ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(rehearsalConf), 0o644)
	_ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(fmt.Sprintf(rehearsalAccounts, "admin", "admin")), 0o644)
	src := fmt.Sprintf("mesh-rehearsal-source-%d", os.Getpid())
	uid := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
	sh(t, "docker", "run", "-d", "--rm", "--name", src, "--user", uid, "-p", "127.0.0.1::4222",
		"-v", filepath.Join(work, "srcdata")+":/data", "-v", conf+":/etc/nats:ro", image, "-c", "/etc/nats/nats.conf")
	t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", src).Run() })
	_, port, _ := strings.Cut(sh(t, "docker", "port", src, "4222/tcp"), ":")
	var nc *nats.Conn
	for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(300 * time.Millisecond) {
		nc, err = nats.Connect("nats://127.0.0.1:"+port, nats.UserInfo("admin", "admin"))
		if err == nil {
			break
		}
		if time.Now().After(deadline) {
			t.Fatal(err)
		}
	}
	defer nc.Close()
	js, _ := nc.JetStream()
	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-rehearsal"})
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 5; i++ {
		if _, err := kv.Put(fmt.Sprintf("k%d", i), []byte("v")); err != nil {
			t.Fatal(err)
		}
	}
	cred, _ := json.Marshal(map[string]string{"url": "nats://127.0.0.1:" + port, "user": "admin", "password": "admin"})
	cmd := exec.Command(filepath.Join(program, "mesh-nats-snapshot"), "snapshot", "--server", "nats://127.0.0.1:"+port, "--credential", "-")
	cmd.Stdin = strings.NewReader(string(cred))
	tarball, err := cmd.Output()
	if err != nil {
		t.Fatalf("the snapshot: %v", err)
	}
	if err := os.WriteFile(filepath.Join(snaps, "bus.tar"), tarball, 0o644); err != nil {
		t.Fatal(err)
	}

	r := LiveRehearsal()
	r.UpgradeOpen = func(context.Context) (bool, string, error) { return false, "", nil }
	r.BusContainer = "mesh-broker-nats-not-here"
	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
	defer cancel()
	got, err := r.Run(ctx, RehearseArgs{Snapshot: filepath.Join(snaps, "bus.tar"), NewImage: image,
		NewProgram: program, NewEntrypoint: entry, OldImage: image})
	if err != nil {
		t.Fatalf("%v\n%+v", err, got)
	}
	if got["ok"] != true {
		t.Fatalf("answer %+v", got)
	}
	if left := sh(t, "docker", "ps", "-a", "-q", "--filter", "name=mesh-rehearsal-new", "--filter", "name=mesh-rehearsal-old"); left != "" {
		t.Errorf("containers are left: %s", left)
	}
}