mesh-controller/cmd/mesh-controller/addresses_test.go

package main

import (
	"encoding/json"
	"os"
	"strings"
	"testing"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/licences"
	"github.com/novox/mesh-controller/internal/link"
	"github.com/novox/mesh-controller/internal/overlay"
)

// An address is read from the node's settings where it is used, never recorded with a port
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.

var aDigest = "sha256:" + strings.Repeat("e", 64)

// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
	manifest, _ := json.Marshal(map[string]any{
		"module": "gitea", "version": "1",
		"resources": []map[string]any{
			{"id": "server", "type": "container", "name": "mesh-gitea",
				"image": "anchor.internal:5100/gitea/server@" + aDigest},
			{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
				"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
			{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
				"image": "valkey/valkey@" + aDigest},
		},
	})
	kept := buildFrom(link.BuildResult{
		ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
		Manifest: manifest,
		Made: []link.MadeArtifact{
			{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
			{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
		},
		Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
	})
	if kept.Module != "gitea" {
		t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
	}
	if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
		t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
	}
	if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
		t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
	}
	recorded, err := catalogue.ParseManifest(kept.Manifest)
	if err != nil {
		t.Fatal(err)
	}
	if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
		t.Errorf("the recorded manifest's image is %v", got)
	}
	if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
		t.Errorf("the recorded manifest's archive is %v", got)
	}
	if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
		t.Errorf("an image the build did not make was rewritten: %v", got)
	}
	if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
		t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
	}
	// What the build stood on is an edge to another module's artifact, and it is recorded the way
	// that artifact is: by path in the store, so the edge still names the same thing when the
	// store answers at another address.
	if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
		t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
	}
}

// aStore is a module offering the artifact store on 5000, published the long way as the
// distribution module does, so a node may be given another number for it.
func aStore() catalogue.Manifest {
	return catalogue.Manifest{Module: "distribution", Version: "1",
		Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
		Serves:   map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
		Listens:  []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
		Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
			"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
}

// **The trust a machine writes for the store, and the address every built image is fetched
// through, say the port the node gave the store** — not the catalogue's number.
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	register(t, open, aStore())
	if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
		map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
		t.Fatal(err)
	}
	// A module the mesh built, recorded by digest and path, running on the other machine.
	if err := open.inventory.RecordBuild(ctx, inventory.Build{
		ID: "b1", Repository: "r", Module: "app", Commit: "abc",
		Made: []inventory.Artifact{{Name: "server", Kind: "image",
			Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
	}); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "app", Version: "1",
		Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
			"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
	if _, err := assign(ctx, open, "laptop", "app"); err != nil {
		t.Fatal(err)
	}
	// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
	// asks where this machine reaches the store, as the docker module does.
	register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
		Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
			"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
	if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
		t.Fatal(err)
	}

	on := map[string]bool{"anchor": true, "laptop": true}
	node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
	if err != nil || !found || node != "anchor" || port != "5101" {
		t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
	}

	var trust string
	for _, r := range composed(t, open, "laptop").Resources {
		if r["path"] == "/etc/docker/daemon.json" {
			trust, _ = r["content"].(string)
		}
		if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
			t.Errorf("the image the mesh built is fetched as %v", r["image"])
		}
	}
	if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
		t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
	}

	// And a replay to the catalogue says where the store is now.
	announced, err := following{open}.Announceable(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
		t.Fatalf("the replay announces %+v", announced)
	}
}

// **The control plane's own connections say the port the node gave the store and the broker.**
//
// Composed from the control plane's own manifest against a real inventory: the store's module is
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
// process is told so beside the sealed connection genesis wrote.
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	raw, err := os.ReadFile("../../module.json")
	if err != nil {
		t.Fatal(err)
	}
	m, err := catalogue.ParseManifest(raw)
	if err != nil {
		t.Fatal(err)
	}
	control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
		Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
	if err != nil {
		t.Fatal(err)
	}
	register(t, open, control)
	register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
		Claims:  []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
		Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
		Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
			"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
	register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
		Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
		Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
			{Port: 5672, From: catalogue.FromMesh}},
		Guards: []int{15672},
		Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
			"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
	// The control plane's own bus user is the installer's, seeded at genesis before the controller
	// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
	if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
		Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
		t.Fatal(err)
	}
	if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
		t.Fatal(err)
	}
	// The store's module is registered and given its port, and NOT assigned: the state genesis
	// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
	if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
		t.Fatal(err)
	}
	if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
		map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
		t.Fatal(err)
	}

	var env map[string]any
	for _, r := range composed(t, open, "anchor").Resources {
		if r["id"] == "mesh-controller.controller" {
			env, _ = r["env"].(map[string]any)
		}
	}
	if env == nil {
		t.Fatal("the control plane's process is not in its own node's declaration")
	}
	for key, want := range map[string]string{
		"MESH_STORE_INVENTORY_PORT": "6852",
		"MESH_STORE_IDENTITY_PORT":  "6852",
		"MESH_STORE_LICENCES_PORT":  "6852",
		"MESH_BROKER_AMQP_PORT":     "5679",
		// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
		// because the sealed value beside it carries the port genesis wrote (finding F3).
		"MESH_BROKER_ADDRESS_PORT":    "",
		"MESH_BROKER_MANAGEMENT_PORT": "",
	} {
		if env[key] != want {
			t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
		}
	}
}

// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
// added here until module.json carries them (the manifest lands one commit after the code that
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
	seatPorts := map[string]string{
		"MESH_STORE_INVENTORY_PORT":   "${seat:mesh-store:5432}",
		"MESH_STORE_IDENTITY_PORT":    "${seat:mesh-store:5432}",
		"MESH_STORE_LICENCES_PORT":    "${seat:mesh-store:5432}",
		"MESH_BROKER_AMQP_PORT":       "${seat:mesh-broker:5672}",
		"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
		"MESH_BROKER_ADDRESS_PORT":    "${seat:mesh-broker:5671}",
	}
	out := m
	out.Resources = nil
	for _, r := range m.Resources {
		if r["type"] != "container" && r["type"] != "process" {
			out.Resources = append(out.Resources, r)
			continue
		}
		copied := map[string]any{}
		for k, v := range r {
			copied[k] = v
		}
		env := map[string]any{}
		if had, ok := r["env"].(map[string]any); ok {
			for k, v := range had {
				env[k] = v
			}
		}
		for k, v := range seatPorts {
			if _, said := env[k]; !said {
				env[k] = v
			}
		}
		copied["env"] = env
		out.Resources = append(out.Resources, copied)
	}
	return out
}

// aLoneNode is one capable machine with nothing placed on any network — the control-node during
// genesis, before the "network" step, which is after the store, the broker, the vault and the
// catalogue have each been built and pushed (finding F2).
func aLoneNode(t *testing.T) *stores {
	t.Helper()
	inventory.ForTest(t)
	licences.ForTest(t)
	open, err := openStores(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(open.Close)
	for _, m := range provided {
		if err := open.inventory.Provide(t.Context(), m); err != nil {
			t.Fatal(err)
		}
	}
	record, err := open.inventory.AddNode(t.Context(), "anchor")
	if err != nil {
		t.Fatal(err)
	}
	reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
		{"name": "container-runtime", "present": true}}})
	var profile map[string]any
	_ = json.Unmarshal(reported, &profile)
	if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
		t.Fatal(err)
	}
	return open
}

// **Before the network exists, the store's own node reaches it by loopback** — never refused,
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
// a node on no network, and builds the catalogue on a base it must be able to pull.
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
	open := aLoneNode(t)
	ctx := t.Context()
	register(t, open, aStore())
	register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
		Requires: []string{catalogue.ArtifactStoreProvision},
		Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
			"image": "registry.example/mesh-builder@" + aDigest}}})
	if err := open.inventory.RecordBuild(ctx, inventory.Build{
		ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
		Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
			Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
	}); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
		Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
			"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
	for _, module := range []string{"distribution", "builder", "postgres"} {
		if _, err := assign(ctx, open, "anchor", module); err != nil {
			t.Fatal(err)
		}
	}
	if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
		map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
		t.Fatal(err)
	}

	var image any
	for _, r := range composed(t, open, "anchor").Resources {
		if r["id"] == "postgres.runtime" {
			image = r["image"]
		}
	}
	if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
		t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
	}
	held := heldBy(ctx)
	if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
		t.Fatalf("a builder beside the store is handed the base %q", got)
	}
}

// **A mesh of one on its network, with no resolver yet, still reaches its store by loopback** (novox/hq issue
// 701). Genesis places the network at its step 21, before anything holds the mesh's resolver seat; the
// filter's build at step 22 was handed anchor.internal:5000, which nothing answered, and stopped. Once a
// resolver holds the seat the store is named by the network's name; a mesh of several is answered as before
// (TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore).
func TestAMeshOfOneIsHandedNoNameUntilAResolverAnswersIt(t *testing.T) {
	open := aLoneNode(t)
	ctx := t.Context()
	// On the private network, as genesis's network step leaves it: placed as the hub, able to run the
	// overlay, its key reported, the overlay assigned.
	if err := open.inventory.SetPlace(ctx, "anchor", "anchor.example:51820", "here", true, "10.77.0.1"); err != nil {
		t.Fatal(err)
	}
	nodes, err := open.inventory.Nodes(ctx)
	if err != nil || len(nodes) != 1 {
		t.Fatalf("nodes: %v %v", nodes, err)
	}
	if err := open.inventory.RecordProfile(ctx, nodes[0].ID, map[string]any{"capabilities": []any{
		map[string]any{"name": "container-runtime", "present": true},
		map[string]any{"name": "wireguard", "present": true},
		map[string]any{"name": "systemd", "present": true}}}); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.RecordOverlayKey(ctx, nodes[0].ID, aPublicKey(t)); err != nil {
		t.Fatal(err)
	}
	if _, err := open.inventory.Assign(ctx, "anchor", overlay.Name); err != nil {
		t.Fatal(err)
	}
	register(t, open, aStore())
	register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
		Requires: []string{catalogue.ArtifactStoreProvision},
		Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
			"image": "registry.example/mesh-builder@" + aDigest},
			// Where the build agent publishes, as its manifest says it.
			{"id": "env", "type": "file", "path": "/etc/mesh-builder.env",
				"content": "MESH_REGISTRY=${bound:" + catalogue.ArtifactStoreProvision + ":at}:${bound:" +
					catalogue.ArtifactStoreProvision + ":port}\n"}}})
	if err := open.inventory.RecordBuild(ctx, inventory.Build{
		ID: "b1", Repository: "r", Module: "mesh-tools", Commit: "abc",
		Made: []inventory.Artifact{{Name: "build", Kind: "image",
			Reference: catalogue.ArtifactStoreScheme + "mesh-tools/build@" + aDigest}},
	}); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "mesh-tools", Version: "1"})
	for _, module := range []string{"distribution", "builder"} {
		if _, err := assign(ctx, open, "anchor", module); err != nil {
			t.Fatal(err)
		}
	}
	if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
		map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
		t.Fatal(err)
	}
	shelf, err := open.inventory.Catalogue(ctx)
	if err != nil {
		t.Fatal(err)
	}
	onNetwork, err := whereEveryoneIs(ctx, open.inventory, shelf)
	if err != nil || len(onNetwork) != 1 {
		t.Fatalf("the lone node is not on the network: %v %v", onNetwork, err)
	}
	if got, err := artifactStoreAddress(ctx, open.inventory, shelf, "anchor"); err != nil || got != "127.0.0.1:5101" {
		t.Fatalf("on the network with no resolver, the mesh of one is handed %q (%v), a name nothing answers", got, err)
	}
	if got := heldBy(ctx)["mesh-tools/build"]; got != "127.0.0.1:5101/mesh-tools/build@"+aDigest {
		t.Fatalf("the builder beside the store is handed the base %q", got)
	}
	publishesTo := func() string {
		t.Helper()
		for _, r := range composed(t, open, "anchor").Resources {
			if r["path"] == "/etc/mesh-builder.env" {
				content, _ := r["content"].(string)
				return content
			}
		}
		t.Fatal("the builder's environment was not composed")
		return ""
	}
	if got := publishesTo(); got != "MESH_REGISTRY=127.0.0.1:5101\n" {
		t.Fatalf("the builder beside the store publishes to %q, a name nothing answers", got)
	}

	// A resolver holds the seat: the name is answered, and the store is named by it.
	if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
		Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
		Claims:   []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}})
	if _, err := assign(ctx, open, "anchor", "resolver"); err != nil {
		t.Fatal(err)
	}
	shelf, err = open.inventory.Catalogue(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if got, err := artifactStoreAddress(ctx, open.inventory, shelf, "anchor"); err != nil || got != "anchor.internal:5101" {
		t.Fatalf("with a resolver answering the mesh's names, the store is handed as %q (%v)", got, err)
	}
	if got := heldBy(ctx)["mesh-tools/build"]; got != "anchor.internal:5101/mesh-tools/build@"+aDigest {
		t.Fatalf("the builder is handed the base %q", got)
	}
	if got := publishesTo(); got != "MESH_REGISTRY=anchor.internal:5101\n" {
		t.Fatalf("with a resolver, the builder publishes to %q", got)
	}
}