A module may need a secret of its own, and the provisioner watches

Two things, both found by trying to write a real postgres module and
discovering it could not be said.

A database has a superuser password, a broker an administrator, a
registry an account. None of them is *for* anybody — they are not the
credential a consumer is given, and the mechanism that hands those out
has a consumer in the middle of it. So a module may declare what it needs
and where to put it, and the mesh generates one per node, seals it, and
reads it no more than it reads any other.

Per node, deliberately: a module running on three machines has three
passwords. One in the manifest instead would put the same secret on every
machine that ever runs it, in a file anybody can read, for ever. Made
once and kept, or a running database would be handed a password it was
not started with; remade when the machine's sealing key changes, like
everything else sealed here.

A need declared and not made is refused rather than skipped, because a
module whose own credential is silently absent starts, fails to
authenticate, and the reason is three layers from the machine reporting
it.

And the provisioner can watch. That is what lets it be a module rather
than a binary somebody places: run once, it needs invoking after every
declaration by a timer or a unit wired to a file; watching, it is an
ordinary long-running service the host already supervises. It polls
rather than watching the filesystem, because the host writes atomically —
the file is replaced, so a watch on the path stops seeing anything after
the first replacement, and a watcher that silently stops working is worse
than a poll. Credentials are compared by digest and never held: this runs
for as long as the machine is up.

internal/inventory/migrations/0012-a-modules-own-secret.sql

@@ -0,0 +1,29 @@
+-- A secret a module needs in order to be itself.
+--
+-- A database has a superuser password, a broker an administrator, a registry an account. None of
+-- them is *for* anybody -- they are not the credential a consumer is given, and the table holding
+-- those has a consumer in its key.
+--
+-- **One per node**, so a module running on three machines has three passwords. A manifest that
+-- carried one instead would put the same secret on every machine that ever runs the module, in a
+-- file anybody can read, for ever.
+--
+-- Sealed to the node before it is written, like everything else here: what is stored is unusable
+-- by whoever holds it, the mesh included.
+
+create table module_secret (
+    node     uuid        not null references node(id) on delete cascade,
+    module   text        not null references module(name) on delete cascade,
+    -- The module's own word for it. Two secrets in one module are ordinary -- a password and a
+    -- token, say -- and telling them apart is the module's business, not the mesh's.
+    name     text        not null,
+
+    sealed   text        not null,
+    -- Which key it was sealed to, so a node that regenerated its key can be told what it can no
+    -- longer open rather than discovering it as a service that will not start.
+    node_key text        not null,
+
+    made_at  timestamptz not null default now(),
+
+    primary key (node, module, name)
+);

internal/inventory/secrets.go

@@ -2,6 +2,7 @@ package inventory
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/novox/mesh-control/internal/secrets"
 )
@@ -133,3 +134,49 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
 	}
 	return out, rows.Err()
 }
+
+// SecretForModule is a secret a module needs in order to be itself, on one machine.
+//
+// Not the credential a consumer is given: a superuser password is not *for* anybody. Made once
+// and kept, because regenerating it on every declaration would change the password a running
+// database has already been started with — and remade when the node's sealing key changes, for
+// the same reason as everything else sealed here.
+func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) {
+	key, err := i.SealingKeyOf(ctx, node)
+	if err != nil {
+		return "", err
+	}
+	if key == "" {
+		return "", fmt.Errorf(
+			"%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
+			module, node)
+	}
+	record, err := i.NodeByName(ctx, node)
+	if err != nil {
+		return "", err
+	}
+
+	var sealed, against string
+	err = i.store.Pool().QueryRow(ctx,
+		`select sealed, node_key from module_secret where node = $1 and module = $2 and name = $3`,
+		record.ID, module, name).Scan(&sealed, &against)
+	if err == nil && against == key {
+		return sealed, nil
+	}
+
+	made, err := secrets.Make(key, key)
+	if err != nil {
+		return "", err
+	}
+	// Sealed once, to one recipient. Make seals to two ends because a provision has two; here
+	// both are the same machine, and only one copy is kept.
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into module_secret (node, module, name, sealed, node_key)
+		 values ($1, $2, $3, $4, $5)
+		 on conflict (node, module, name) do update set
+		   sealed = excluded.sealed, node_key = excluded.node_key, made_at = now()`,
+		record.ID, module, name, made.ForConsumer, key); err != nil {
+		return "", err
+	}
+	return made.ForConsumer, nil
+}

internal/inventory/secrets_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/ecdh"
 	"crypto/rand"
 	"encoding/base64"
+	"github.com/novox/mesh-control/internal/catalogue"
 	"strings"
 	"testing"
 
@@ -230,3 +231,97 @@ func TestSecretsGoWhenANodeLeaves(t *testing.T) {
 		t.Fatalf("%d credential(s) outlived the machine they were for", left)
 	}
 }
+
+func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) {
+	// A module running on three machines has three passwords. One in the manifest instead would
+	// put the same secret on every machine that ever runs it, in a file anybody can read.
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if here == there {
+		t.Fatal("two machines were given the same secret")
+	}
+
+	// Made once and kept, or a running database would be handed a password it was not started
+	// with on the next declaration.
+	again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again != here {
+		t.Fatal("asking twice made a second secret")
+	}
+}
+
+func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if one == two {
+		t.Fatal("two names gave one secret")
+	}
+}
+
+func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) {
+	// The node generated a new sealing key and can no longer open what was sealed to the old one.
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node, err := inv.NodeByName(ctx, "consumer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fresh, _ := aSealingKey(t)
+	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
+		t.Fatal(err)
+	}
+	after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after == before {
+		t.Fatal("a machine was handed a secret sealed to a key it no longer has")
+	}
+}
+
+func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	if _, err := inv.AddNode(ctx, "bare"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil {
+		t.Fatal("a secret was made for a machine that cannot open one")
+	}
+}