package bootstrap import ( "context" "strings" "testing" "time" ) // Step 9 is where the control plane stops being a special case. These tests defend the two things // that could go wrong quietly: pinning it to the wrong image, and delivering it store connections // the mesh invented rather than the ones the foundation actually made. // theControlPlaneModule is the manifest the build produces at step 3: the control plane's own, // from the root of its repository, its artifact resolved to the image the machine built — named by // the digest of its own configuration, with no registry in front (novox/hq ADR 0069). // // A fixture rather than the file itself, unlike the foundation example the rewrite tests use: the // control plane is a different repository on a different branch, and a test that read it would // pass or fail according to what somebody else had checked out. What it must stay faithful to is // the SHAPE — the built image's bare id, the own-secret per context, the mount from the machine's // path to the container's, and the environment file that fills what is not a path. const theControlPlaneModule = `{ "module": "mesh-controller", "version": "1", "slug": "control", "capabilities": ["container-runtime"], "claims": [{"name": "the-controller", "scope": "mesh"}], "own-secrets": { "inventory": "/var/lib/mesh/mesh-controller/inventory", "identity": "/var/lib/mesh/mesh-controller/identity", "licences": "/var/lib/mesh/mesh-controller/licences", "broker": "/var/lib/mesh/mesh-controller/broker", "broker-management": "/var/lib/mesh/mesh-controller/broker-management" }, "resources": [ {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"}, {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env", "mode": "0600", "content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, {"id": "server", "type": "container", "name": "mesh-controller", "image": "` + builtImage + `", "network": "host", "args": ["serve"], "env-file": ["/var/lib/mesh/mesh-controller/broker.env"], "env": { "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", "MESH_STORE_IDENTITY_FILE": "/run/secrets/identity", "MESH_STORE_LICENCES_FILE": "/run/secrets/licences", "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" }, "volumes": [ "mesh-broker-tls:/broker-tls:ro", "/var/lib/mesh/mesh-controller/inventory:/run/secrets/inventory:ro", "/var/lib/mesh/mesh-controller/identity:/run/secrets/identity:ro", "/var/lib/mesh/mesh-controller/licences:/run/secrets/licences:ro" ]} ] }` const pushedReference = "127.0.0.1:5000/mesh-controller@sha256:" + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" // builtImage is what step 3 built, as the machine that built it names it. const builtImage = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" // theBuild is what step 3 hands step 9. func theBuild(manifest string) Built { return Built{Module: "mesh-controller", Commit: "a1b2c3d4", Image: builtImage, Manifest: []byte(manifest)} } // **The whole reference moves.** The build names its image by the bare digest of its configuration, // which only the machine that built it can resolve; the mesh's record must name what the registry // assigned, `/@sha256:…`, or every other machine the module is pushed to // would go looking for an image nothing serves. func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) { pinned, places, err := pinImage([]byte(theControlPlaneModule), builtImage, pushedReference, "mesh-controller") if err != nil { t.Fatal(err) } if places != 1 { t.Errorf("the built image was found in %d place(s)", places) } if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) { t.Errorf("the manifest does not name the pushed image:\n%s", pinned) } if strings.Contains(string(pinned), builtImage) { t.Errorf("the built image's bare id survived, which no other machine can resolve:\n%s", pinned) } } // A manifest naming some other image describes some other build. Registering it would install a // control plane that is not the image this machine just published, which is the one thing this // step exists to guarantee. func TestAManifestNamingAnotherBuildIsRefused(t *testing.T) { other := strings.Replace(theControlPlaneModule, builtImage, "sha256:"+strings.Repeat("9", 64), 1) if _, _, err := pinImage([]byte(other), builtImage, pushedReference, "mesh-controller"); err == nil { t.Fatal("a manifest naming some other image was accepted") } } // Every place moves. A manifest naming its image in a second resource — a migrate step beside the // server — would otherwise be left half pinned, and fail inside an apply rather than here. func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { twice := strings.Replace(theControlPlaneModule, `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},`, `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"}, {"id": "migrate", "type": "container", "name": "mesh-controller-migrate", "run-once": true, "image": "`+builtImage+`", "args": ["migrate"]},`, 1) pinned, places, err := pinImage([]byte(twice), builtImage, pushedReference, "mesh-controller") if err != nil { t.Fatal(err) } if places != 2 { t.Errorf("the built image was found in %d place(s), and the manifest names it twice", places) } if strings.Contains(string(pinned), builtImage) { t.Error("the built image's id survived the pinning") } } // **The connections are the foundation's, and they are read out of the bundle that made them.** // The mesh cannot invent them: they are the credentials the foundation created the databases with, // and thirty-two random bytes in their place would leave the control plane unable to open a single // context. The pairing is read from the manifest so that whatever the catalogue calls these // secrets is what is delivered. func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { wanted, err := secretsByVariableIn([]byte(theControlPlaneModule)) if err != nil { t.Fatal(err) } // **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads // them at /run/secrets. Matching on the path alone would find nothing and refuse a correct // manifest, which is exactly the ordinary case in the catalogue. for variable, secret := range map[string]string{ "MESH_STORE_INVENTORY": "inventory", "MESH_STORE_IDENTITY": "identity", "MESH_STORE_LICENCES": "licences", "MESH_BROKER_AMQP": "broker", "MESH_BROKER_MANAGEMENT": "broker-management", } { if wanted[variable] != secret { t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret) } } // And what the manifest fills from the machine rather than from a secret is left alone. if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed { t.Error("the address the mesh composes from the machine was treated as a secret") } // The values are the foundation's own, taken from the produced bundle rather than composed. rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } runtime := &asked{answer: aMeshThatAgrees(nil)} control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second} delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, []byte(theControlPlaneModule), rewritten.Declaration, func(string) {}) if err != nil { t.Fatal(err) } // Three stores and both halves of the broker: everything the foundation made and nothing else. if len(delivered) != 5 { t.Fatalf("%d values were delivered, and the foundation names five: %v", len(delivered), delivered) } for _, secret := range delivered { if !runtime.ran("secret accept anchor mesh-controller " + secret + " --from") { t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands) } } } // A secret the foundation did not make is left for the mesh to make, and said so. Every other // secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh. func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) { extra := strings.Replace(theControlPlaneModule, `"broker": "/var/lib/mesh/mesh-controller/broker",`, `"broker": "/var/lib/mesh/mesh-controller/broker", "something-new": "/var/lib/mesh/mesh-controller/something-new",`, 1) extra = strings.Replace(extra, `"content": "MESH_BROKER_AMQP=${secret:broker}\n`, `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } runtime := &asked{answer: aMeshThatAgrees(nil)} control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second} var said []string delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, []byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) }) if err != nil { t.Fatal(err) } for _, secret := range delivered { if secret == "something-new" { t.Error("a value the foundation never made was accepted as though it had") } } if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") { t.Errorf("nothing was said about the secret the mesh has to make: %v", said) } } // A manifest whose container reads a file no own-secret writes is refused. The mesh would seal // nothing there and the control plane would find an empty file where a connection string has to // be — which presents as a control plane that will not start, three steps from the cause. func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { mismatched := strings.Replace(theControlPlaneModule, `"inventory": "/var/lib/mesh/mesh-controller/inventory",`, `"inventory": "/var/lib/mesh/mesh-controller/somewhere-else",`, 1) _, err := secretsByVariableIn([]byte(mismatched)) if err == nil { t.Fatal("a manifest whose two ends do not meet was accepted") } if !strings.Contains(err.Error(), "own-secret") { t.Errorf("the refusal does not say which half is missing: %v", err) } } // A manifest asking for no store connections at all describes a control plane that can open // nothing, and the refusal says what shape the installer delivers into — because the manifest is // written in another repository and this is where the two have to agree. func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) { rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } runtime := &asked{answer: aMeshThatAgrees(nil)} control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second} bare := `{"module":"mesh-controller","version":"1","resources":[ {"id":"container","type":"container","name":"mesh-controller", "image":"` + builtImage + `"}]}` _, err = deliverStores(context.Background(), Options{Node: "anchor"}, control, []byte(bare), rewritten.Declaration, func(string) {}) if err == nil { t.Fatal("a control plane that can open nothing was accepted") } if !strings.Contains(err.Error(), storeVariablePrefix+""+storeFileSuffix) { t.Errorf("the refusal does not say what shape is expected: %v", err) } } // The permanent control plane is asked a question, not merely looked at — the same question the // temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so // a reply proves the sealed connections it was given are the ones the foundation made. func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) { runtime := &asked{answer: aMeshThatAgrees(map[string]string{ "module list": "", "exec mesh-controller /mesh-controller": "1 node, 0 waiting\n", })} control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second} rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } // No catalogue: the control plane's manifest is the one the build produced (novox/hq // 04-ISSUES/072), and step 9 reads nothing from the catalogue any more. out, err := InstallControlPlane(context.Background(), installing(t, t.TempDir()), Deps{Run: runtime.run}, control, rewritten.Declaration, theBuild(theControlPlaneModule), pushedReference, func(string) {}) if err != nil { t.Fatal(err) } if out.Answered != "1 node, 0 waiting" { t.Errorf("the permanent control plane's reply is reported as %q", out.Answered) } if !runtime.ran("docker exec mesh-controller " + controlPlaneBinary + " status") { t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands) } // And the module was registered with the digest, not with the placeholder. if !runtime.ran("module add /mesh-controller-module.json") { t.Errorf("the module was never registered: %v", runtime.commands) } } // The broker settings take the file shape too (novox/hq ADR 0086): a `MESH_BROKER_…_FILE` pointing // at a mounted own-secret is delivered as that secret, exactly as a store connection is. func TestABrokerSettingReadFromAFileIsDeliveredToo(t *testing.T) { manifest := `{ "module": "mesh-controller", "version": "1", "own-secrets": {"broker": "/var/lib/mesh/mesh-controller/broker", "inventory": "/var/lib/mesh/mesh-controller/inventory"}, "resources": [{ "id": "server", "type": "container", "name": "mesh-controller", "image": "x@sha256:0", "volumes": ["/var/lib/mesh/mesh-controller/broker:/run/secrets/broker:ro", "/var/lib/mesh/mesh-controller/inventory:/run/secrets/inventory:ro"], "env": {"MESH_BROKER_AMQP_FILE": "/run/secrets/broker", "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"} }]}` wanted, err := secretsByVariableIn([]byte(manifest)) if err != nil { t.Fatal(err) } if wanted["MESH_BROKER_AMQP"] != "broker" || wanted["MESH_STORE_INVENTORY"] != "inventory" { t.Fatalf("wanted %v", wanted) } if _, has := wanted["MESH_BROKER_CERTIFICATE"]; has { t.Fatal("a plain path variable was taken for a secret") } }