package bootstrap import ( "context" "encoding/json" "fmt" "net/http" "os" "path/filepath" "strings" "testing" "time" "github.com/novox/mesh-host/internal/declaration" ) // aCatalogue is a catalogue checkout holding these modules' manifests. func aCatalogue(t *testing.T, manifests map[string]string) string { t.Helper() dir := t.TempDir() for module, manifest := range manifests { at := filepath.Join(dir, catalogueDir, module) if err := os.MkdirAll(at, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil { t.Fatal(err) } } return dir } // aBusModule is the nats module's manifest as far as adoption reads it: its server container, the // directories it mounts, its certificate through an access, and its user list — the shape of the // catalogue's modules/nats/module.json. func aBusModule(l busLayout) string { return `{"module": "nats", "version": "1", "claims": [{"name": "mesh-broker", "scope": "mesh"}], "bus-users": "` + l.Users + `", "resources": [ {"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"], "volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}], "accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}], "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}` } // theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats). var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats", Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls", Users: "/var/lib/nats-module/conf/accounts.conf"} // theModulesImage is what the bus runs once the applier recreated it on the module's built image. const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333" // aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the // module's image, and its monitor says ok. func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) { runtime := &asked{answer: aMeshThatAgrees(map[string]string{ "inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n", })} return runtime, func(_ context.Context, url string) (int, string, error) { if url != busMonitor { return 0, "", fmt.Errorf("asked %s", url) } return http.StatusOK, `{"status":"ok"}`, nil } } func busStepWith(t *testing.T, o Options, runtime *asked, fetch func(context.Context, string) (int, string, error)) ([]string, error) { t.Helper() aGenesisBusOnDisk(t) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} var said []string err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch, func(line string) { said = append(said, line) }) return said, err } func busStep(t *testing.T, catalogue string) (*asked, error) { t.Helper() runtime, fetch := aBusThatAdopts() o := installing(t, catalogue) o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} _, err := busStepWith(t, o, runtime, fetch) return runtime, err } // The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module // the mesh goes on managing, registered from the catalogue — never lavinmq, which left it. func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) { catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) runtime, err := busStep(t, catalogue) if err != nil { t.Fatal(err) } if !runtime.ran("module add /nats-module.json") { t.Fatalf("the nats module was not registered: %v", runtime.commands) } if runtime.ran("lavinmq") { t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands) } } // A catalogue that holds no nats module fails the step by name, before anything is registered — // which is what today's step did for lavinmq, silently adopting a module that no longer exists. func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) { catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`}) runtime, err := busStep(t, catalogue) if err == nil { t.Fatal("a catalogue without the nats module was adopted from") } if !strings.Contains(err.Error(), "nats module") { t.Errorf("the refusal does not name the module it looked for: %v", err) } if runtime.ran("module add") { t.Errorf("something was registered: %v", runtime.commands) } } // A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus // beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every // difference with both sides, before anything is registered. The layout is the one the bundle had // before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590). func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) { before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf", Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"} catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)}) runtime, err := busStep(t, catalogue) if err == nil { t.Fatal("a bus module kept elsewhere was adopted") } for _, named := range []string{ `"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`, `"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`, `"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`, } { if !strings.Contains(err.Error(), named) { t.Errorf("the refusal does not name %s: %v", named, err) } } if runtime.ran("module add") { t.Errorf("something was registered: %v", runtime.commands) } } // One field moved is refused as surely as all of them: a module whose streams are elsewhere would be // raised empty on the bus's own name. func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) { moved := theModulesLayout moved.Streams = "/srv/nats" catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)}) runtime, err := busStep(t, catalogue) if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") { t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err) } if strings.Contains(err.Error(), "certificate") { t.Errorf("the refusal names what agrees: %v", err) } if runtime.ran("module add") { t.Errorf("something was registered: %v", runtime.commands) } } // The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step // gives it the setting that maps that to the port the bundle published, 5671 unless given — before the // push, so the first declaration already names it. func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) { for _, c := range []struct { given int want string }{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} { catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) runtime, fetch := aBusThatAdopts() o := installing(t, catalogue) o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} o.Ports = FoundationPorts{Bus: c.given} said, err := busStepWith(t, o, runtime, fetch) if err != nil { t.Fatal(err) } set, pushed := -1, -1 for i, command := range runtime.commands { if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 { set = i } if strings.Contains(command, "push anchor") && pushed < 0 { pushed = i } } if set < 0 || pushed < 0 || set > pushed { t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands) } if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) { t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n")) } } } // The one restart is said before it happens, and the step goes on only once the bus runs the module's // image and answers (novox/hq ADR 0333 §3). func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) { previous := answerEvery answerEvery = time.Millisecond defer func() { answerEvery = previous }() catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image // The push is sent; the node takes a few passes to recreate the bus, and the server a few more to // answer. pushed, inspected, monitored := false, 0, 0 runtime := &asked{} agrees := aMeshThatAgrees(nil) runtime.answer = func(name string, args []string) (string, error) { joined := strings.Join(args, " ") switch { case strings.Contains(joined, "bus upgrade --why"): pushed = true case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): if !pushed { return bundleImage + "\n", nil } inspected++ if inspected < 3 { return bundleImage + "\n", nil } return theModulesImage + "\n", nil } return agrees(name, args) } fetch := func(context.Context, string) (int, string, error) { monitored++ if monitored < 3 { return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil } return http.StatusOK, `{"status":"ok"}`, nil } o := installing(t, catalogue) o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} o.Wait = 10 * time.Second said, err := busStepWith(t, o, runtime, fetch) if err != nil { t.Fatal(err) } text := strings.Join(said, "\n") restart := strings.Index(text, "restarting mesh-broker-nats — once") answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage) adopted := strings.Index(text, "adopted mesh-broker-nats") if restart < 0 || answers < restart || adopted < answers { t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text) } if inspected < 3 || monitored < 3 { t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+ "times, its monitor %d)", inspected, monitored) } } // A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after // this one reaches the mesh through that bus, and the push was not applied. func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) { previous := answerEvery answerEvery = time.Millisecond defer func() { answerEvery = previous }() catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image runtime := &asked{answer: aMeshThatAgrees(map[string]string{ "inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n", })} fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil } o := installing(t, catalogue) o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} o.Wait = 20 * time.Millisecond _, err = busStepWith(t, o, runtime, fetch) if err == nil { t.Fatal("a bus the node never recreated was taken as adopted") } if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") { t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err) } } // A re-run after the adoption finds the bus on the module's image already, and says so rather than // announcing a restart that does not happen. func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) { catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) runtime, fetch := aBusThatAdopts() o := installing(t, catalogue) o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} said, err := busStepWith(t, o, runtime, fetch) if err != nil { t.Fatal(err) } text := strings.Join(said, "\n") if strings.Contains(text, "restarting") || !strings.Contains(text, "already adopted mesh-broker-nats") { t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text) } } // theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333, // "How it is checked"): in a merge check the clone the build seat puts beside this repository // (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue // beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check // never skips it. func theCataloguesBusModule(t *testing.T) []byte { t.Helper() const inside = "mesh-catalog/modules/nats/module.json" if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" { raw, err := os.ReadFile(filepath.Join(root, inside)) if err != nil { t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+ "cannot be held to the module's: %v", root, err) } t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside)) return raw } beside := filepath.Join("..", "..", "..", inside) raw, err := os.ReadFile(beside) if err != nil { t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+ "bus is not held to the nats module's manifest here; the merge check holds it", beside) } t.Logf("judged against the catalogue checked out at %s", beside) return raw } // **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's // manifest: the bus's container, its streams, its configuration, its certificate and its user list are // where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised // `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590). func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) { manifest := theCataloguesBusModule(t) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil { t.Fatal(err) } // And the port the installer's setting maps is the one the module declares (ADR 0333 §2). var m struct { Resources []struct { Type string `json:"type"` Ports []string `json:"ports"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { t.Fatal(err) } declared := false for _, r := range m.Resources { for _, p := range r.Ports { if p == fmt.Sprintf("%d:4222", busModulePort) { declared = true } } } if !declared { t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+ "to the node's bus port", busModulePort) } } // The bundle as the installer reads it agrees with the module's layout as this repository states it, // and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the // bus reads, and makes the container that reads it re-read it. func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) { rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } ours, err := foundationBusLayout(rewritten.Declaration) if err != nil { t.Fatal(err) } if ours != theModulesLayout { t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout) } if ours.Users != busAccounts || ours.Container != busContainer { t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s", busAccounts, busContainer, ours.Users, ours.Container) } if filepath.Dir(ours.Users) != ours.Conf { t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf) } } // The bundle's bus configuration and user list are seeds (create-once): an installer run again after // the adoption applies the bundle again, and must not put the bundle's one user back over the users the // module writes there (ADR 0333 §4). func TestTheBundlesBusFilesAreSeeds(t *testing.T) { rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } seeds := map[string]bool{} for _, r := range rewritten.Declaration.Resources { if f, ok := r.(*declaration.File); ok && f.CreateOnce { seeds[f.Path] = true } } for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} { if !seeds[path] { t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path) } } } // aGenesisBusOnDisk puts what the adoption reads on the machine in a test's own directory: the bus's // certificate, the controller's bus credential, and where the snapshot goes. The step's wait is short. func aGenesisBusOnDisk(t *testing.T) { t.Helper() dir := t.TempDir() cert := filepath.Join(dir, "tls.crt") if err := os.WriteFile(cert, []byte(aCertificatePEM), 0o644); err != nil { t.Fatal(err) } secret := filepath.Join(dir, "bus-controller.secret") if err := os.WriteFile(secret, []byte("BUS-PW"), 0o600); err != nil { t.Fatal(err) } was := []string{busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret} wasWithin := busStepWithin busSnapshotFile, busSnapshotCredential = filepath.Join(dir, "bus.tar"), filepath.Join(dir, "bus.credential") busCertificateFile, busControllerSecret = cert, secret busStepWithin = 50 * time.Millisecond t.Cleanup(func() { busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret = was[0], was[1], was[2], was[3] busStepWithin = wasWithin }) } // aCertificatePEM is a PEM block standing for the bus's certificate; its pin is the sha256 of these bytes. const aCertificatePEM = "-----BEGIN CERTIFICATE-----\nAAECAwQFBgcICQ==\n-----END CERTIFICATE-----\n"