jochen
a3e8a4185b
`assign` recorded a module and `push` sealed a random own secret where its bus credential belongs; the process crash-looped until a person ran `module issue` and pushed again, and the only warning was one line in a list printed on every push. Now assigning a module that declares a broker secret issues the credential in the same act — kept when one exists, so re-assigning rotates nothing — and when the bus cannot be reached from here the assignment says which verb to run. A push never seals a placeholder in a credential's place: a module whose bus user is unminted is refused by name, with the verb. The control plane's own user is the installer's, seeded at genesis, which the test now says. And what reads one of a module's own secrets is restarted when it changes — composed for a container or daemon that names the secret's path in its volumes, environment or env-files, so a manifest need not say it: the build machine ran on an hour-old credential because its manifest restarted it on its environment file alone (issue 206). A scheduled or run-once process is left alone; it reads afresh.
345 lines
14 KiB
Go
345 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/licences"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// An address is read from the node's settings where it is used, never recorded with a port
|
|
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
|
|
|
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
|
|
|
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
|
|
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
|
|
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
|
manifest, _ := json.Marshal(map[string]any{
|
|
"module": "gitea", "version": "1",
|
|
"resources": []map[string]any{
|
|
{"id": "server", "type": "container", "name": "mesh-gitea",
|
|
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
|
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
|
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
|
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
|
"image": "valkey/valkey@" + aDigest},
|
|
},
|
|
})
|
|
kept := buildFrom(link.BuildResult{
|
|
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
|
Manifest: manifest,
|
|
Made: []link.MadeArtifact{
|
|
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
|
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
|
},
|
|
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
|
})
|
|
if kept.Module != "gitea" {
|
|
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
|
}
|
|
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
|
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
|
}
|
|
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
|
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
|
}
|
|
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
|
t.Errorf("the recorded manifest's image is %v", got)
|
|
}
|
|
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
|
t.Errorf("the recorded manifest's archive is %v", got)
|
|
}
|
|
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
|
t.Errorf("an image the build did not make was rewritten: %v", got)
|
|
}
|
|
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
|
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
|
}
|
|
// What the build stood on is an edge to another module's artifact, and it is recorded the way
|
|
// that artifact is: by path in the store, so the edge still names the same thing when the
|
|
// store answers at another address.
|
|
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
|
|
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
|
|
}
|
|
}
|
|
|
|
// aStore is a module offering the artifact store on 5000, published the long way as the
|
|
// distribution module does, so a node may be given another number for it.
|
|
func aStore() catalogue.Manifest {
|
|
return catalogue.Manifest{Module: "distribution", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
|
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
|
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
|
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
|
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
|
}
|
|
|
|
// **The trust a machine writes for the store, and the address every built image is fetched
|
|
// through, say the port the node gave the store** — not the catalogue's number.
|
|
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
register(t, open, aStore())
|
|
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
|
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A module the mesh built, recorded by digest and path, running on the other machine.
|
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
|
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
|
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
|
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
|
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
on := map[string]bool{"anchor": true, "laptop": true}
|
|
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
|
if err != nil || !found || node != "anchor" || port != "5101" {
|
|
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
|
}
|
|
|
|
var trust string
|
|
for _, r := range composed(t, open, "laptop").Resources {
|
|
if r["path"] == "/etc/docker/daemon.json" {
|
|
trust, _ = r["content"].(string)
|
|
}
|
|
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
|
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
|
}
|
|
}
|
|
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
|
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
|
}
|
|
|
|
// And a replay to the catalogue says where the store is now.
|
|
announced, err := following{open}.Announceable(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
|
t.Fatalf("the replay announces %+v", announced)
|
|
}
|
|
}
|
|
|
|
// **The control plane's own connections say the port the node gave the store and the broker.**
|
|
//
|
|
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
|
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
|
// container is told so beside the sealed connection genesis wrote.
|
|
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
raw, err := os.ReadFile("../../module.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, err := catalogue.ParseManifest(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
|
Reference: "registry.example/control@" + aDigest}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
register(t, open, control)
|
|
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
|
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
|
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
|
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
|
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
|
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
|
{Port: 5672, From: catalogue.FromMesh}},
|
|
Guards: []int{15672},
|
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
|
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
|
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
|
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
|
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
|
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
|
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
|
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
|
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var env map[string]any
|
|
for _, r := range composed(t, open, "anchor").Resources {
|
|
if r["id"] == "mesh-controller.server" {
|
|
env, _ = r["env"].(map[string]any)
|
|
}
|
|
}
|
|
if env == nil {
|
|
t.Fatal("the control plane's container is not in its own node's declaration")
|
|
}
|
|
for key, want := range map[string]string{
|
|
"MESH_STORE_INVENTORY_PORT": "6852",
|
|
"MESH_STORE_IDENTITY_PORT": "6852",
|
|
"MESH_STORE_LICENCES_PORT": "6852",
|
|
"MESH_BROKER_AMQP_PORT": "5679",
|
|
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
|
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
|
"MESH_BROKER_ADDRESS_PORT": "",
|
|
"MESH_BROKER_MANAGEMENT_PORT": "",
|
|
} {
|
|
if env[key] != want {
|
|
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
|
// added here until module.json carries them (the manifest lands one commit after the code that
|
|
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
|
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
|
seatPorts := map[string]string{
|
|
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
|
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
|
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
|
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
|
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
|
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
|
}
|
|
out := m
|
|
out.Resources = nil
|
|
for _, r := range m.Resources {
|
|
if r["type"] != "container" {
|
|
out.Resources = append(out.Resources, r)
|
|
continue
|
|
}
|
|
copied := map[string]any{}
|
|
for k, v := range r {
|
|
copied[k] = v
|
|
}
|
|
env := map[string]any{}
|
|
if had, ok := r["env"].(map[string]any); ok {
|
|
for k, v := range had {
|
|
env[k] = v
|
|
}
|
|
}
|
|
for k, v := range seatPorts {
|
|
if _, said := env[k]; !said {
|
|
env[k] = v
|
|
}
|
|
}
|
|
copied["env"] = env
|
|
out.Resources = append(out.Resources, copied)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
|
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
|
// catalogue have each been built and pushed (finding F2).
|
|
func aLoneNode(t *testing.T) *stores {
|
|
t.Helper()
|
|
inventory.ForTest(t)
|
|
licences.ForTest(t)
|
|
open, err := openStores(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(open.Close)
|
|
for _, m := range provided {
|
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
|
{"name": "container-runtime", "present": true}}})
|
|
var profile map[string]any
|
|
_ = json.Unmarshal(reported, &profile)
|
|
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return open
|
|
}
|
|
|
|
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
|
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
|
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
|
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
|
open := aLoneNode(t)
|
|
ctx := t.Context()
|
|
register(t, open, aStore())
|
|
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
|
Requires: []string{catalogue.ArtifactStoreProvision},
|
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
|
"image": "registry.example/mesh-builder@" + aDigest}}})
|
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
|
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
|
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
|
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
|
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
|
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
|
for _, module := range []string{"distribution", "builder", "postgres"} {
|
|
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
|
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var image any
|
|
for _, r := range composed(t, open, "anchor").Resources {
|
|
if r["id"] == "postgres.runtime" {
|
|
image = r["image"]
|
|
}
|
|
}
|
|
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
|
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
|
}
|
|
held := heldBy(ctx)
|
|
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
|
t.Fatalf("a builder beside the store is handed the base %q", got)
|
|
}
|
|
}
|