diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index 281f305..d1cea2d 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -336,7 +336,23 @@ func declarationWith(ctx context.Context, open *stores, node string, needed := map[string]map[string]string{} for _, m := range plan.Modules { for name := range m.OwnSecrets { - sealed, err := inv.SecretForModule(ctx, node, m.Module, name) + // Minted on the send path and only read on every other. Making one is an insert, and + // a question that writes is a question that can block against the machine it is about. + var sealed string + var err error + if choosing == Allocating { + sealed, err = inv.SecretForModule(ctx, node, m.Module, name) + } else { + var held bool + sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name) + if err == nil && !held { + // Never issued, so this machine cannot be running it. Left out rather than + // invented: an empty string here would compose a declaration that differs + // from what would be sent, and the comparison this feeds would then be + // answering about a declaration nothing will ever push. + continue + } + } if err != nil { return nil, err } diff --git a/examples/modules/README.md b/examples/modules/README.md index 1c129bc..e62cf03 100644 --- a/examples/modules/README.md +++ b/examples/modules/README.md @@ -43,7 +43,7 @@ is `mesh0` on every machine, and the address a resolver listens on for the machi ## Asking for a bucket -`object-store.json` provides one, `photos.json` asks for one. Together they are the whole of an +`minio.json` provides one, `photos.json` asks for one. Together they are the whole of an edge, and they are here as a **pair** because that is the only way to see the halves line up: | the provider says | the consumer says | diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index c152f61..488acba 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -220,12 +220,17 @@ func TestTheResolverDoesNotAskItselfForUpstreams(t *testing.T) { // The two halves of an object-store edge, as a pair. // +// `minio.json` is the provider. There were two manifests describing the same object store — the +// other named `object-store.json` — which is not a choice between implementations but one module +// written twice: same image, same provision, same scope. Assigning both to a node would have +// collided on `s3-bucket`. +// // A provider and a consumer that only ever appear separately are two manifests nobody has checked // against each other: the name one provides has to be the name the other requires, and the key a // consumer contributes has to be the one the provisioner reads. Both were got wrong while writing // them, and neither would have been caught by parsing either file alone. func TestTheObjectStoreEdgeFitsTogether(t *testing.T) { - provider := read(t, "object-store.json") + provider := read(t, "minio.json") consumer := read(t, "photos.json") const provision = "s3-bucket" diff --git a/examples/modules/object-store.json b/examples/modules/object-store.json deleted file mode 100644 index 5d2d3df..0000000 --- a/examples/modules/object-store.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "module": "object-store", - "version": "1", - "provides": [ - { - "name": "s3-bucket", - "scope": "mesh" - } - ], - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 9000, - "protocol": "tcp", - "from": "mesh", - "why": "the S3 endpoint, for modules on any machine that were granted a bucket" - } - ], - "serves": { - "s3-bucket": { - "scheme": "http", - "region": "us-east-1" - } - }, - "receives": { - "s3-bucket": "/var/lib/objectstore/grants" - }, - "grants": { - "s3-bucket": "/var/lib/objectstore/grants" - }, - "own-secrets": { - "root": "/var/lib/objectstore/root.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/objectstore", - "mode": "0700" - }, - { - "id": "grants", - "type": "directory", - "path": "/var/lib/objectstore/grants", - "mode": "0700" - }, - { - "id": "store", - "type": "container", - "name": "mesh-store", - "image": "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2", - "args": [ - "server", - "/data" - ], - "env": { - "MINIO_ROOT_USER": "meshroot" - }, - "ports": [ - "9000" - ], - "volumes": [ - "mesh-store-data:/data", - "/var/lib/objectstore/root.secret:/run/secrets/root:ro" - ] - }, - { - "id": "provisioner", - "type": "container", - "name": "mesh-provision-objectstore", - "image": "mesh-provision-objectstore@sha256:0000000000000000000000000000000000000000000000000000000000000000", - "env": { - "GRANTS": "/var/lib/objectstore/grants", - "MESH_OBJECTSTORE_URL": "http://127.0.0.1:9000", - "MESH_OBJECTSTORE_ROOT_USER": "meshroot", - "MESH_OBJECTSTORE_ROOT_PASSWORD_FILE": "/run/secrets/root" - }, - "volumes": [ - "/var/lib/objectstore/grants:/var/lib/objectstore/grants:ro", - "/var/lib/objectstore/root.secret:/run/secrets/root:ro" - ] - } - ] -} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 4d4d50b..8672542 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -2,8 +2,11 @@ package inventory import ( "context" + "errors" "fmt" + "github.com/jackc/pgx/v5" + "github.com/novox/mesh-control/internal/secrets" ) @@ -150,6 +153,46 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret, // and kept, because regenerating it on every declaration would change the password a running // database has already been started with — and remade when the node's sealing key changes, for // the same reason as everything else sealed here. +// ModuleSecretIfIssued is what a module already holds on a node, and nothing if it holds nothing. +// +// **The read half of SecretForModule**, which mints one when there is none — an insert, and a row +// lock, on a path that also serves questions. Composing a declaration to answer *is this machine +// running what I would send it* went through the minting version for every module on every node, +// so asking wrote to the database and blocked against the machine it was asking about. +// +// A module with no secret yet has never been sent one, which is the same answer the caller wanted +// anyway: this machine is not running what the mesh would send it. +func (i *Inventory) ModuleSecretIfIssued( + ctx context.Context, node, module, name string, +) (string, bool, error) { + key, err := i.SealingKeyOf(ctx, node) + if err != nil || key == "" { + return "", false, err + } + record, err := i.NodeByName(ctx, node) + if err != nil { + return "", false, err + } + var sealed, against, origin string + err = i.store.Pool().QueryRow(ctx, + `select sealed, node_key, origin from module_secret + where node = $1 and module = $2 and name = $3`, + record.ID, module, name).Scan(&sealed, &against, &origin) + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + // Sealed to a key the node no longer has is not something it holds. Reported as absent rather + // than as an error: this is the read, and refusing here would make a question fail for a + // condition its writing counterpart is the right place to explain. + if against != key { + return "", false, nil + } + return sealed, true, nil +} + func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) { key, err := i.SealingKeyOf(ctx, node) if err != nil {