mesh-host/internal/bootstrap/control_test.go

package bootstrap

import (
	"context"
	"strings"
	"testing"
	"time"
)

// Step 9 is where the control plane stops being a special case. These tests defend the two things
// that could go wrong quietly: pinning it to the wrong image, and delivering it store connections
// the mesh invented rather than the ones the foundation actually made.

// theControlPlaneModule is the manifest the build produces at step 3: the control plane's own,
// from the root of its repository, its artifact resolved to the image the machine built — named by
// the digest of its own configuration, with no registry in front (novox/hq ADR 0069).
//
// A fixture rather than the file itself, unlike the foundation example the rewrite tests use: the
// control plane is a different repository on a different branch, and a test that read it would
// pass or fail according to what somebody else had checked out. What it must stay faithful to is
// the SHAPE — the built image's bare id, the own-secret per context, the mount from the machine's
// path to the container's, and the environment file that fills what is not a path.
const theControlPlaneModule = `{
  "module": "mesh-controller",
  "version": "1",
  "slug": "control",
  "capabilities": ["container-runtime"],
  "claims": [{"name": "the-controller", "scope": "mesh"}],
  "own-secrets": {
    "inventory": "/var/lib/mesh/mesh-controller/inventory",
    "identity": "/var/lib/mesh/mesh-controller/identity",
    "licences": "/var/lib/mesh/mesh-controller/licences",
    "bus": "/var/lib/mesh/mesh-controller/bus"
  },
  "resources": [
    {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},
    {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env",
     "mode": "0600",
     "content": "MESH_BUS_NATS=${secret:bus}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
    {"id": "server", "type": "container", "name": "mesh-controller",
     "image": "` + builtImage + `",
     "network": "host", "args": ["serve"],
     "env-file": ["/var/lib/mesh/mesh-controller/broker.env"],
     "env": {
       "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
       "MESH_STORE_IDENTITY_FILE": "/run/secrets/identity",
       "MESH_STORE_LICENCES_FILE": "/run/secrets/licences",
       "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
     },
     "volumes": [
       "mesh-broker-tls:/broker-tls:ro",
       "/var/lib/mesh/mesh-controller/inventory:/run/secrets/inventory:ro",
       "/var/lib/mesh/mesh-controller/identity:/run/secrets/identity:ro",
       "/var/lib/mesh/mesh-controller/licences:/run/secrets/licences:ro"
     ]}
  ]
}`

const pushedReference = "127.0.0.1:5000/mesh-controller@sha256:" +
	"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"

// builtImage is what step 3 built, as the machine that built it names it.
const builtImage = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"

// theBuild is what step 3 hands step 9.
func theBuild(manifest string) Built {
	return Built{Module: "mesh-controller", Commit: "a1b2c3d4", Image: builtImage, Manifest: []byte(manifest)}
}

// **The whole reference moves.** The build names its image by the bare digest of its configuration,
// which only the machine that built it can resolve; the mesh's record must name what the registry
// assigned, `<registry>/<repository>@sha256:…`, or every other machine the module is pushed to
// would go looking for an image nothing serves.
func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) {
	pinned, places, err := pinImage([]byte(theControlPlaneModule), builtImage, pushedReference, "mesh-controller")
	if err != nil {
		t.Fatal(err)
	}
	if places != 1 {
		t.Errorf("the built image was found in %d place(s)", places)
	}
	if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) {
		t.Errorf("the manifest does not name the pushed image:\n%s", pinned)
	}
	if strings.Contains(string(pinned), builtImage) {
		t.Errorf("the built image's bare id survived, which no other machine can resolve:\n%s", pinned)
	}
}

// A manifest naming some other image describes some other build. Registering it would install a
// control plane that is not the image this machine just published, which is the one thing this
// step exists to guarantee.
func TestAManifestNamingAnotherBuildIsRefused(t *testing.T) {
	other := strings.Replace(theControlPlaneModule, builtImage, "sha256:"+strings.Repeat("9", 64), 1)
	if _, _, err := pinImage([]byte(other), builtImage, pushedReference, "mesh-controller"); err == nil {
		t.Fatal("a manifest naming some other image was accepted")
	}
}

// Every place moves. A manifest naming its image in a second resource — a migrate step beside the
// server — would otherwise be left half pinned, and fail inside an apply rather than here.
func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) {
	twice := strings.Replace(theControlPlaneModule,
		`{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},`,
		`{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},
     {"id": "migrate", "type": "container", "name": "mesh-controller-migrate", "run-once": true,
      "image": "`+builtImage+`", "args": ["migrate"]},`, 1)

	pinned, places, err := pinImage([]byte(twice), builtImage, pushedReference, "mesh-controller")
	if err != nil {
		t.Fatal(err)
	}
	if places != 2 {
		t.Errorf("the built image was found in %d place(s), and the manifest names it twice", places)
	}
	if strings.Contains(string(pinned), builtImage) {
		t.Error("the built image's id survived the pinning")
	}
}

// **The connections are the foundation's, and they are read out of the bundle that made them.**
// The mesh cannot invent them: they are the credentials the foundation created the databases with,
// and thirty-two random bytes in their place would leave the control plane unable to open a single
// context. The pairing is read from the manifest so that whatever the catalogue calls these
// secrets is what is delivered.
func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
	wanted, err := secretsByVariableIn([]byte(theControlPlaneModule))
	if err != nil {
		t.Fatal(err)
	}
	// **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads
	// them at /run/secrets. Matching on the path alone would find nothing and refuse a correct
	// manifest, which is exactly the ordinary case in the catalogue.
	for variable, secret := range map[string]string{
		"MESH_STORE_INVENTORY": "inventory",
		"MESH_STORE_IDENTITY":  "identity",
		"MESH_STORE_LICENCES":  "licences",
		"MESH_BUS_NATS":        "bus",
	} {
		if wanted[variable] != secret {
			t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret)
		}
	}
	// And what the manifest fills from the machine rather than from a secret is left alone.
	if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed {
		t.Error("the address the mesh composes from the machine was treated as a secret")
	}

	// The values are the foundation's own, taken from the produced bundle rather than composed.
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	runtime := &asked{answer: aMeshThatAgrees(nil)}
	control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}

	delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control,
		[]byte(theControlPlaneModule), rewritten.Declaration, func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	// Three stores and the controller's account on the bus: everything the foundation made and
	// nothing else.
	if len(delivered) != 4 {
		t.Fatalf("%d values were delivered, and the foundation names four: %v",
			len(delivered), delivered)
	}
	for _, secret := range delivered {
		if !runtime.ran("secret accept anchor mesh-controller " + secret + " --from") {
			t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands)
		}
	}
}

// A secret the foundation did not make is left for the mesh to make, and said so. Every other
// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh.
func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) {
	extra := strings.Replace(theControlPlaneModule,
		`"bus": "/var/lib/mesh/mesh-controller/bus"`,
		`"bus": "/var/lib/mesh/mesh-controller/bus",
    "something-new": "/var/lib/mesh/mesh-controller/something-new"`, 1)
	extra = strings.Replace(extra,
		`"content": "MESH_BUS_NATS=${secret:bus}\n`,
		`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BUS_NATS=${secret:bus}\n`, 1)
	if extra == theControlPlaneModule {
		t.Fatal("the fixture no longer says what this test adds to")
	}

	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	runtime := &asked{answer: aMeshThatAgrees(nil)}
	control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}

	var said []string
	delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control,
		[]byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) })
	if err != nil {
		t.Fatal(err)
	}
	for _, secret := range delivered {
		if secret == "something-new" {
			t.Error("a value the foundation never made was accepted as though it had")
		}
	}
	if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") {
		t.Errorf("nothing was said about the secret the mesh has to make: %v", said)
	}
}

// A manifest whose container reads a file no own-secret writes is refused. The mesh would seal
// nothing there and the control plane would find an empty file where a connection string has to
// be — which presents as a control plane that will not start, three steps from the cause.
func TestAConnectionFileNothingWritesIsRefused(t *testing.T) {
	mismatched := strings.Replace(theControlPlaneModule,
		`"inventory": "/var/lib/mesh/mesh-controller/inventory",`,
		`"inventory": "/var/lib/mesh/mesh-controller/somewhere-else",`, 1)

	_, err := secretsByVariableIn([]byte(mismatched))
	if err == nil {
		t.Fatal("a manifest whose two ends do not meet was accepted")
	}
	if !strings.Contains(err.Error(), "own-secret") {
		t.Errorf("the refusal does not say which half is missing: %v", err)
	}
}

// A manifest asking for no store connections at all describes a control plane that can open
// nothing, and the refusal says what shape the installer delivers into — because the manifest is
// written in another repository and this is where the two have to agree.
func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) {
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	runtime := &asked{answer: aMeshThatAgrees(nil)}
	control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}

	bare := `{"module":"mesh-controller","version":"1","resources":[
	  {"id":"container","type":"container","name":"mesh-controller",
	   "image":"` + builtImage + `"}]}`

	_, err = deliverStores(context.Background(), Options{Node: "anchor"}, control,
		[]byte(bare), rewritten.Declaration, func(string) {})
	if err == nil {
		t.Fatal("a control plane that can open nothing was accepted")
	}
	if !strings.Contains(err.Error(), storeVariablePrefix+"<CONTEXT>"+storeFileSuffix) {
		t.Errorf("the refusal does not say what shape is expected: %v", err)
	}
}

// The permanent control plane is asked a question, not merely looked at — the same question the
// temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so
// a reply proves the sealed connections it was given are the ones the foundation made.
func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) {
	runtime := &asked{answer: aMeshThatAgrees(map[string]string{
		"module list":                           "",
		"exec mesh-controller /mesh-controller": "1 node, 0 waiting\n",
	})}
	control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}

	// No catalogue: the control plane's manifest is the one the build produced (novox/hq
	// 04-ISSUES/072), and step 9 reads nothing from the catalogue any more.
	out, err := InstallControlPlane(context.Background(), installing(t, t.TempDir()),
		Deps{Run: runtime.run}, control, rewritten.Declaration, theBuild(theControlPlaneModule),
		pushedReference, func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	if out.Answered != "1 node, 0 waiting" {
		t.Errorf("the permanent control plane's reply is reported as %q", out.Answered)
	}
	if !runtime.ran("docker exec mesh-controller " + controlPlaneBinary + " status") {
		t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands)
	}
	// And the module was registered with the digest, not with the placeholder.
	if !runtime.ran("module add /mesh-controller-module.json") {
		t.Errorf("the module was never registered: %v", runtime.commands)
	}
}

// The broker settings take the file shape too (novox/hq ADR 0086): a `MESH_BROKER_…_FILE` pointing
// at a mounted own-secret is delivered as that secret, exactly as a store connection is.
func TestABrokerSettingReadFromAFileIsDeliveredToo(t *testing.T) {
	manifest := `{
	  "module": "mesh-controller", "version": "1",
	  "own-secrets": {"broker": "/var/lib/mesh/mesh-controller/broker", "inventory": "/var/lib/mesh/mesh-controller/inventory"},
	  "resources": [{
	    "id": "server", "type": "container", "name": "mesh-controller", "image": "x@sha256:0",
	    "volumes": ["/var/lib/mesh/mesh-controller/broker:/run/secrets/broker:ro",
	                "/var/lib/mesh/mesh-controller/inventory:/run/secrets/inventory:ro"],
	    "env": {"MESH_BROKER_AMQP_FILE": "/run/secrets/broker", "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
	            "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"}
	  }]}`
	wanted, err := secretsByVariableIn([]byte(manifest))
	if err != nil {
		t.Fatal(err)
	}
	if wanted["MESH_BROKER_AMQP"] != "broker" || wanted["MESH_STORE_INVENTORY"] != "inventory" {
		t.Fatalf("wanted %v", wanted)
	}
	if _, has := wanted["MESH_BROKER_CERTIFICATE"]; has {
		t.Fatal("a plain path variable was taken for a secret")
	}
}

// Step 9 with nothing built is a caller's fault, and it stops rather than pretending to pivot.
func TestTheControlPlaneCannotBeInstalledWithoutABuild(t *testing.T) {
	runtime := &asked{answer: aMeshThatAgrees(map[string]string{})}
	control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}
	rewritten, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	_, err = InstallControlPlane(context.Background(), installing(t, t.TempDir()),
		Deps{Run: runtime.run}, control, rewritten.Declaration, Built{}, pushedReference, func(string) {})
	if err == nil || !strings.Contains(err.Error(), "was not built") {
		t.Fatalf("a missing build was not refused: %v", err)
	}
	if runtime.ran("module add") {
		t.Error("something was registered without a manifest")
	}
}