mesh-host/internal/bootstrap/phase3_test.go

package bootstrap

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// aCatalogue is a catalogue checkout holding these modules' manifests.
func aCatalogue(t *testing.T, manifests map[string]string) string {
	t.Helper()
	dir := t.TempDir()
	for module, manifest := range manifests {
		at := filepath.Join(dir, catalogueDir, module)
		if err := os.MkdirAll(at, 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

// aBusModule is the nats module's manifest as far as adoption reads it: its server container, the
// directories it mounts, its certificate through an access, and its user list — the shape of the
// catalogue's modules/nats/module.json.
func aBusModule(l busLayout) string {
	return `{"module": "nats", "version": "1",
  "claims": [{"name": "mesh-broker", "scope": "mesh"}],
  "bus-users": "` + l.Users + `",
  "resources": [
    {"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"],
     "volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}],
  "accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}],
  "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
}

// theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats).
var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats",
	Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls",
	Users: "/var/lib/nats-module/conf/accounts.conf"}

// theModulesImage is what the bus runs once the applier recreated it on the module's built image.
const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333"

// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the
// module's image, and its monitor says ok.
func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) {
	runtime := &asked{answer: aMeshThatAgrees(map[string]string{
		"inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n",
	})}
	return runtime, func(_ context.Context, url string) (int, string, error) {
		if url != busMonitor {
			return 0, "", fmt.Errorf("asked %s", url)
		}
		return http.StatusOK, `{"status":"ok"}`, nil
	}
}

func busStepWith(t *testing.T, o Options, runtime *asked,
	fetch func(context.Context, string) (int, string, error)) ([]string, error) {
	t.Helper()
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
	var said []string
	err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch,
		func(line string) { said = append(said, line) })
	return said, err
}

func busStep(t *testing.T, catalogue string) (*asked, error) {
	t.Helper()
	runtime, fetch := aBusThatAdopts()
	o := installing(t, catalogue)
	o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
	_, err := busStepWith(t, o, runtime, fetch)
	return runtime, err
}

// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
	runtime, err := busStep(t, catalogue)
	if err != nil {
		t.Fatal(err)
	}
	if !runtime.ran("module add /nats-module.json") {
		t.Fatalf("the nats module was not registered: %v", runtime.commands)
	}
	if runtime.ran("lavinmq") {
		t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands)
	}
}

// A catalogue that holds no nats module fails the step by name, before anything is registered —
// which is what today's step did for lavinmq, silently adopting a module that no longer exists.
func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
	catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`})
	runtime, err := busStep(t, catalogue)
	if err == nil {
		t.Fatal("a catalogue without the nats module was adopted from")
	}
	if !strings.Contains(err.Error(), "nats module") {
		t.Errorf("the refusal does not name the module it looked for: %v", err)
	}
	if runtime.ran("module add") {
		t.Errorf("something was registered: %v", runtime.commands)
	}
}

// A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus
// beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every
// difference with both sides, before anything is registered. The layout is the one the bundle had
// before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590).
func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) {
	before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf",
		Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"}
	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)})
	runtime, err := busStep(t, catalogue)
	if err == nil {
		t.Fatal("a bus module kept elsewhere was adopted")
	}
	for _, named := range []string{
		`"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`,
		`"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`,
		`"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`,
	} {
		if !strings.Contains(err.Error(), named) {
			t.Errorf("the refusal does not name %s: %v", named, err)
		}
	}
	if runtime.ran("module add") {
		t.Errorf("something was registered: %v", runtime.commands)
	}
}

// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be
// raised empty on the bus's own name.
func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) {
	moved := theModulesLayout
	moved.Streams = "/srv/nats"
	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)})
	runtime, err := busStep(t, catalogue)
	if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") {
		t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err)
	}
	if strings.Contains(err.Error(), "certificate") {
		t.Errorf("the refusal names what agrees: %v", err)
	}
	if runtime.ran("module add") {
		t.Errorf("something was registered: %v", runtime.commands)
	}
}

// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step
// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the
// push, so the first declaration already names it.
func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) {
	for _, c := range []struct {
		given int
		want  string
	}{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} {
		catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
		runtime, fetch := aBusThatAdopts()
		o := installing(t, catalogue)
		o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
		o.Ports = FoundationPorts{Bus: c.given}
		said, err := busStepWith(t, o, runtime, fetch)
		if err != nil {
			t.Fatal(err)
		}
		set, pushed := -1, -1
		for i, command := range runtime.commands {
			if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 {
				set = i
			}
			if strings.Contains(command, "push anchor") && pushed < 0 {
				pushed = i
			}
		}
		if set < 0 || pushed < 0 || set > pushed {
			t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands)
		}
		if !strings.Contains(strings.Join(said, "\n"), "settings          nats on anchor: "+c.want) {
			t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n"))
		}
	}
}

// The one restart is said before it happens, and the step goes on only once the bus runs the module's
// image and answers (novox/hq ADR 0333 §3).
func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image

	// The push is sent; the node takes a few passes to recreate the bus, and the server a few more to
	// answer.
	pushed, inspected, monitored := false, 0, 0
	runtime := &asked{}
	agrees := aMeshThatAgrees(nil)
	runtime.answer = func(name string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "push anchor"):
			pushed = true
		case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
			if !pushed {
				return bundleImage + "\n", nil
			}
			inspected++
			if inspected < 3 {
				return bundleImage + "\n", nil
			}
			return theModulesImage + "\n", nil
		}
		return agrees(name, args)
	}
	fetch := func(context.Context, string) (int, string, error) {
		monitored++
		if monitored < 3 {
			return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil
		}
		return http.StatusOK, `{"status":"ok"}`, nil
	}
	o := installing(t, catalogue)
	o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
	o.Wait = 10 * time.Second
	said, err := busStepWith(t, o, runtime, fetch)
	if err != nil {
		t.Fatal(err)
	}
	text := strings.Join(said, "\n")
	restart := strings.Index(text, "restarting        mesh-broker-nats — once")
	answers := strings.Index(text, "bus answers       mesh-broker-nats on "+theModulesImage)
	adopted := strings.Index(text, "adopted           mesh-broker-nats")
	if restart < 0 || answers < restart || adopted < answers {
		t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text)
	}
	if inspected < 3 || monitored < 3 {
		t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+
			"times, its monitor %d)", inspected, monitored)
	}
}

// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after
// this one reaches the mesh through that bus, and the push was not applied.
func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
	runtime := &asked{answer: aMeshThatAgrees(map[string]string{
		"inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n",
	})}
	fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil }
	o := installing(t, catalogue)
	o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
	o.Wait = 20 * time.Millisecond
	_, err = busStepWith(t, o, runtime, fetch)
	if err == nil {
		t.Fatal("a bus the node never recreated was taken as adopted")
	}
	if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") {
		t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err)
	}
}

// A re-run after the adoption finds the bus on the module's image already, and says so rather than
// announcing a restart that does not happen.
func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) {
	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
	runtime, fetch := aBusThatAdopts()
	o := installing(t, catalogue)
	o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
	said, err := busStepWith(t, o, runtime, fetch)
	if err != nil {
		t.Fatal(err)
	}
	text := strings.Join(said, "\n")
	if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") {
		t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text)
	}
}

// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333,
// "How it is checked"): in a merge check the clone the build seat puts beside this repository
// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue
// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check
// never skips it.
func theCataloguesBusModule(t *testing.T) []byte {
	t.Helper()
	const inside = "mesh-catalog/modules/nats/module.json"
	if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
		raw, err := os.ReadFile(filepath.Join(root, inside))
		if err != nil {
			t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+
				"cannot be held to the module's: %v", root, err)
		}
		t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside))
		return raw
	}
	beside := filepath.Join("..", "..", "..", inside)
	raw, err := os.ReadFile(beside)
	if err != nil {
		t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+
			"bus is not held to the nats module's manifest here; the merge check holds it", beside)
	}
	t.Logf("judged against the catalogue checked out at %s", beside)
	return raw
}

// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's
// manifest: the bus's container, its streams, its configuration, its certificate and its user list are
// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised
// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590).
func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) {
	manifest := theCataloguesBusModule(t)
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil {
		t.Fatal(err)
	}
	// And the port the installer's setting maps is the one the module declares (ADR 0333 §2).
	var m struct {
		Resources []struct {
			Type  string   `json:"type"`
			Ports []string `json:"ports"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		t.Fatal(err)
	}
	declared := false
	for _, r := range m.Resources {
		for _, p := range r.Ports {
			if p == fmt.Sprintf("%d:4222", busModulePort) {
				declared = true
			}
		}
	}
	if !declared {
		t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+
			"to the node's bus port", busModulePort)
	}
}

// The bundle as the installer reads it agrees with the module's layout as this repository states it,
// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the
// bus reads, and makes the container that reads it re-read it.
func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) {
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	ours, err := foundationBusLayout(rewritten.Declaration)
	if err != nil {
		t.Fatal(err)
	}
	if ours != theModulesLayout {
		t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout)
	}
	if ours.Users != busAccounts || ours.Container != busContainer {
		t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s",
			busAccounts, busContainer, ours.Users, ours.Container)
	}
	if filepath.Dir(ours.Users) != ours.Conf {
		t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf)
	}
}

// The bundle's bus configuration and user list are seeds (create-once): an installer run again after
// the adoption applies the bundle again, and must not put the bundle's one user back over the users the
// module writes there (ADR 0333 §4).
func TestTheBundlesBusFilesAreSeeds(t *testing.T) {
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	seeds := map[string]bool{}
	for _, r := range rewritten.Declaration.Resources {
		if f, ok := r.(*declaration.File); ok && f.CreateOnce {
			seeds[f.Path] = true
		}
	}
	for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} {
		if !seeds[path] {
			t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path)
		}
	}
}