From 4d6ec5b10cf917188b41e1b08780f2774e84bb09 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 1 Sep 2026 21:06:17 +0200 Subject: [PATCH] One object store, not two MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit object-store.json and minio.json described the same thing: same image, same provision at the same scope, same provisioner. Not two implementations a person could choose between — one module written twice. Assigning both to a node would have collided on `s3-bucket`. It exists because it was written first, to pair with photos.json for the README's worked edge, and minio.json was the fuller version of the same module written later. Nobody removed the first. The pair test keeps its point and now reads the surviving one. Checked across the rest: this was the only duplicate. --- cmd/mesh-control/plan.go | 18 ++++++- examples/modules/README.md | 2 +- examples/modules/modules_test.go | 7 ++- examples/modules/object-store.json | 86 ------------------------------ internal/inventory/secrets.go | 43 +++++++++++++++ 5 files changed, 67 insertions(+), 89 deletions(-) delete mode 100644 examples/modules/object-store.json diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index 281f305..d1cea2d 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -336,7 +336,23 @@ func declarationWith(ctx context.Context, open *stores, node string, needed := map[string]map[string]string{} for _, m := range plan.Modules { for name := range m.OwnSecrets { - sealed, err := inv.SecretForModule(ctx, node, m.Module, name) + // Minted on the send path and only read on every other. Making one is an insert, and + // a question that writes is a question that can block against the machine it is about. + var sealed string + var err error + if choosing == Allocating { + sealed, err = inv.SecretForModule(ctx, node, m.Module, name) + } else { + var held bool + sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name) + if err == nil && !held { + // Never issued, so this machine cannot be running it. Left out rather than + // invented: an empty string here would compose a declaration that differs + // from what would be sent, and the comparison this feeds would then be + // answering about a declaration nothing will ever push. + continue + } + } if err != nil { return nil, err } diff --git a/examples/modules/README.md b/examples/modules/README.md index 1c129bc..e62cf03 100644 --- a/examples/modules/README.md +++ b/examples/modules/README.md @@ -43,7 +43,7 @@ is `mesh0` on every machine, and the address a resolver listens on for the machi ## Asking for a bucket -`object-store.json` provides one, `photos.json` asks for one. Together they are the whole of an +`minio.json` provides one, `photos.json` asks for one. Together they are the whole of an edge, and they are here as a **pair** because that is the only way to see the halves line up: | the provider says | the consumer says | diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index c152f61..488acba 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -220,12 +220,17 @@ func TestTheResolverDoesNotAskItselfForUpstreams(t *testing.T) { // The two halves of an object-store edge, as a pair. // +// `minio.json` is the provider. There were two manifests describing the same object store — the +// other named `object-store.json` — which is not a choice between implementations but one module +// written twice: same image, same provision, same scope. Assigning both to a node would have +// collided on `s3-bucket`. +// // A provider and a consumer that only ever appear separately are two manifests nobody has checked // against each other: the name one provides has to be the name the other requires, and the key a // consumer contributes has to be the one the provisioner reads. Both were got wrong while writing // them, and neither would have been caught by parsing either file alone. func TestTheObjectStoreEdgeFitsTogether(t *testing.T) { - provider := read(t, "object-store.json") + provider := read(t, "minio.json") consumer := read(t, "photos.json") const provision = "s3-bucket" diff --git a/examples/modules/object-store.json b/examples/modules/object-store.json deleted file mode 100644 index 5d2d3df..0000000 --- a/examples/modules/object-store.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "module": "object-store", - "version": "1", - "provides": [ - { - "name": "s3-bucket", - "scope": "mesh" - } - ], - "capabilities": [ - "container-runtime" - ], - "listens": [ - { - "port": 9000, - "protocol": "tcp", - "from": "mesh", - "why": "the S3 endpoint, for modules on any machine that were granted a bucket" - } - ], - "serves": { - "s3-bucket": { - "scheme": "http", - "region": "us-east-1" - } - }, - "receives": { - "s3-bucket": "/var/lib/objectstore/grants" - }, - "grants": { - "s3-bucket": "/var/lib/objectstore/grants" - }, - "own-secrets": { - "root": "/var/lib/objectstore/root.secret" - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/objectstore", - "mode": "0700" - }, - { - "id": "grants", - "type": "directory", - "path": "/var/lib/objectstore/grants", - "mode": "0700" - }, - { - "id": "store", - "type": "container", - "name": "mesh-store", - "image": "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2", - "args": [ - "server", - "/data" - ], - "env": { - "MINIO_ROOT_USER": "meshroot" - }, - "ports": [ - "9000" - ], - "volumes": [ - "mesh-store-data:/data", - "/var/lib/objectstore/root.secret:/run/secrets/root:ro" - ] - }, - { - "id": "provisioner", - "type": "container", - "name": "mesh-provision-objectstore", - "image": "mesh-provision-objectstore@sha256:0000000000000000000000000000000000000000000000000000000000000000", - "env": { - "GRANTS": "/var/lib/objectstore/grants", - "MESH_OBJECTSTORE_URL": "http://127.0.0.1:9000", - "MESH_OBJECTSTORE_ROOT_USER": "meshroot", - "MESH_OBJECTSTORE_ROOT_PASSWORD_FILE": "/run/secrets/root" - }, - "volumes": [ - "/var/lib/objectstore/grants:/var/lib/objectstore/grants:ro", - "/var/lib/objectstore/root.secret:/run/secrets/root:ro" - ] - } - ] -} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 4d4d50b..8672542 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -2,8 +2,11 @@ package inventory import ( "context" + "errors" "fmt" + "github.com/jackc/pgx/v5" + "github.com/novox/mesh-control/internal/secrets" ) @@ -150,6 +153,46 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret, // and kept, because regenerating it on every declaration would change the password a running // database has already been started with — and remade when the node's sealing key changes, for // the same reason as everything else sealed here. +// ModuleSecretIfIssued is what a module already holds on a node, and nothing if it holds nothing. +// +// **The read half of SecretForModule**, which mints one when there is none — an insert, and a row +// lock, on a path that also serves questions. Composing a declaration to answer *is this machine +// running what I would send it* went through the minting version for every module on every node, +// so asking wrote to the database and blocked against the machine it was asking about. +// +// A module with no secret yet has never been sent one, which is the same answer the caller wanted +// anyway: this machine is not running what the mesh would send it. +func (i *Inventory) ModuleSecretIfIssued( + ctx context.Context, node, module, name string, +) (string, bool, error) { + key, err := i.SealingKeyOf(ctx, node) + if err != nil || key == "" { + return "", false, err + } + record, err := i.NodeByName(ctx, node) + if err != nil { + return "", false, err + } + var sealed, against, origin string + err = i.store.Pool().QueryRow(ctx, + `select sealed, node_key, origin from module_secret + where node = $1 and module = $2 and name = $3`, + record.ID, module, name).Scan(&sealed, &against, &origin) + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + // Sealed to a key the node no longer has is not something it holds. Reported as absent rather + // than as an error: this is the read, and refusing here would make a question fail for a + // condition its writing counterpart is the right place to explain. + if against != key { + return "", false, nil + } + return sealed, true, nil +} + func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) { key, err := i.SealingKeyOf(ctx, node) if err != nil {