diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 1417763..f49b78b 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -1265,8 +1265,23 @@ func declarationWith(ctx context.Context, inv *inventory.Inventory, node string, if err != nil { return nil, err } - return plan.Declaration( - catalogue.Rendering{Settings: settings, Generators: gens, Grants: grants}) + // And each module's own secrets — a superuser password, an administrator, an account. Made + // per node, so a module running on three machines has three. + needed := map[string]map[string]string{} + for _, m := range plan.Modules { + for name := range m.Needs { + sealed, err := inv.SecretForModule(ctx, node, m.Module, name) + if err != nil { + return nil, err + } + if needed[m.Module] == nil { + needed[m.Module] = map[string]string{} + } + needed[m.Module][name] = sealed + } + } + return plan.Declaration(catalogue.Rendering{ + Settings: settings, Generators: gens, Grants: grants, Needed: needed}) } // grantsFor is every credential this node must create, because something elsewhere uses it. diff --git a/examples/README.md b/examples/README.md index 4c06917..48c1302 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,3 +10,16 @@ with the module that ships the software it configures. | | | |---|---| | `postgres-provisioner` | the last step of a credential: reads what the mesh delivered and makes PostgreSQL accept it | + +## Running the provisioner + +`--watch` reconciles now and again whenever what the mesh delivered changes. That is what lets it +be a module: an ordinary long-running service the host supervises, rather than something that has +to be invoked after every declaration by a timer or a unit wired to a file. + +It polls rather than watching the filesystem, because the host writes atomically — the file is +replaced, so a watch on the path stops seeing anything after the first replacement. A watcher that +silently stops working is worse than a poll. + +Credentials are compared by digest and never by content. This runs for as long as the machine is +up, and a secret does not belong in a long-lived variable when a hash answers the same question. diff --git a/examples/postgres-provisioner/Dockerfile b/examples/postgres-provisioner/Dockerfile new file mode 100644 index 0000000..54562b5 --- /dev/null +++ b/examples/postgres-provisioner/Dockerfile @@ -0,0 +1,18 @@ +# The provisioner, as a module ships one. +# +# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on +# it. Static and FROM scratch for the same reason the control plane's image is: it is fetched by +# digest and run on a machine, and everything in it is something a person would have to audit. +FROM golang:1.25-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \ + -o /mesh-provision-postgres ./examples/postgres-provisioner + +FROM scratch +COPY --from=build /mesh-provision-postgres /mesh-provision-postgres +# Watching by default, because that is what makes it a module: an ordinary long-running service +# the host supervises, rather than something invoked after every declaration. +ENTRYPOINT ["/mesh-provision-postgres", "--watch"] diff --git a/examples/postgres-provisioner/main.go b/examples/postgres-provisioner/main.go index 4792ef9..3c522d6 100644 --- a/examples/postgres-provisioner/main.go +++ b/examples/postgres-provisioner/main.go @@ -20,12 +20,17 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" + "os/signal" "path/filepath" "sort" "strings" + "syscall" + "time" "github.com/jackc/pgx/v5" ) @@ -53,12 +58,99 @@ type manifest struct { } func main() { - if err := run(context.Background()); err != nil { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // Once, or whenever what it was given changes. + // + // **Watching is what lets this be a module.** Run once, it has to be invoked by something + // after every declaration — a timer that runs it when nothing changed, or a unit wired to + // restart on a file. Watching, it is an ordinary long-running service, which is a shape the + // mesh already delivers and the host already supervises. + // + // The file it watches is the manifest the mesh writes. A credential changing rewrites the + // file beside it and not the manifest, so the manifest is stamped whenever either is written + // — which is why this compares content rather than modification time. + if len(os.Args) > 1 && os.Args[1] == "--watch" { + if err := watch(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err) + os.Exit(1) + } + return + } + if err := run(ctx); err != nil { fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err) os.Exit(1) } } +// watch reconciles now, and again whenever what the mesh delivered changes. +// +// By polling rather than by watching the filesystem, because the file is replaced rather than +// written in place — the host writes atomically, so an inotify watch on the path stops seeing +// anything after the first replacement, which is a watcher that silently stops working. +func watch(ctx context.Context) error { + const every = 10 * time.Second + var last string + + for { + state, err := given() + switch { + case err != nil: + // Said and retried. A provisioner that exits because the mesh has not written + // anything yet is one that has to be restarted by hand after the first push. + fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err) + case state != last: + if err := run(ctx); err != nil { + // Reported and retried. The usual reason is that the database has not finished + // starting, and giving up would mean a module that works only if the two + // containers happen to come up in the right order. + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + last = state + } + } + + select { + case <-ctx.Done(): + return nil + case <-time.After(every): + } + } +} + +// given is everything the mesh has delivered, as one string, so a change of any of it is one +// comparison. +// +// The credentials are included by their **digest**, never their content: this is compared, logged +// on nothing, and held in memory for as long as the process runs, and a secret does not belong in +// any of that when a hash answers the same question. +func given() (string, error) { + grants := os.Getenv("GRANTS") + if grants == "" { + grants = "/var/lib/postgres/grants" + } + entries, err := os.ReadDir(grants) + if err != nil { + return "", err + } + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + sort.Strings(names) + + sum := sha256.New() + for _, name := range names { + body, err := os.ReadFile(filepath.Join(grants, name)) + if err != nil { + return "", err + } + fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body)) + } + return hex.EncodeToString(sum.Sum(nil)), nil +} + func run(ctx context.Context) error { grants := os.Getenv("GRANTS") if grants == "" { diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index c2e1339..c82e61f 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -44,6 +44,10 @@ type Grant struct { // Rendering is everything needed to turn a resolution into the declaration a node is sent. type Rendering struct { + // Needed is each module's own secrets, sealed to this node, keyed by module and then by the + // name the module gave it. + Needed map[string]map[string]string + Settings SettingsBy Generators map[string]Generator // Grants are the credentials this node must create, for the provisions it offers. Passed in @@ -74,6 +78,19 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { var out []map[string]any for _, m := range r.Modules { resources := m.Resources + for _, name := range sortedKeys(m.Needs) { + sealed := with.Needed[m.Module][name] + if sealed == "" { + // Declared and not made. Refused rather than skipped: a module whose own + // credential is silently absent starts, fails to authenticate, and the reason is + // three layers away from the machine reporting it. + return nil, fmt.Errorf( + "%s needs a secret called %q and none was made for it", m.Module, name) + } + resources = append(append([]map[string]any{}, resources...), map[string]any{ + "id": NeedID(name), "type": "file", "path": m.Needs[name], "sealed": sealed, + }) + } for _, to := range sortedKeys(m.Secrets) { var found *Needed for i, n := range r.Needs { diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 0539ad9..482c4b4 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -199,6 +199,19 @@ type Manifest struct { // makes `restart-on` precise. Secrets map[string]string `json:"secrets,omitempty"` + // Needs is a secret this module needs for itself, and where to put it. + // + // Not tied to a consumer. A database has a superuser password, a broker has an administrator, + // a registry has an account — each is a secret the module needs in order to be itself, and + // none of them is *for* anybody. Keyed by a name of the module's choosing, valued by the file + // it lands in. + // + // **Generated per node and sealed to it**, like everything else the mesh hands out, so a + // module running on three machines has three passwords and the mesh can read none of them. A + // manifest carrying one instead would put the same secret on every machine that ever runs the + // module, in a file anybody can read, for ever. + Needs map[string]string `json:"needs,omitempty"` + // Grants is a directory this module wants the credentials of its consumers written into, per // provision it offers — one file per consumer, named for it, holding the value alone. // @@ -235,6 +248,9 @@ const ( ArtifactArchive = "archive" ) +// NeedID is the resource identity of the file a module's own secret lands in. +func NeedID(name string) string { return "needs-" + name } + // SecretID is the resource identity of the file a module is given a credential in. func SecretID(requirement string) string { return "secret-" + requirement } @@ -374,6 +390,15 @@ func ParseManifest(raw []byte) (Manifest, error) { "%s binds %q and does not require it", m.Module, to)) } } + for name, where := range m.Needs { + if !strings.HasPrefix(where, "/") { + problems = append(problems, fmt.Sprintf( + "%s needs %q at %q, which is not an absolute path", m.Module, name, where)) + } + if name == "" { + problems = append(problems, m.Module+" needs a secret with no name") + } + } for to, where := range m.Secrets { if !strings.HasPrefix(where, "/") { problems = append(problems, fmt.Sprintf( diff --git a/internal/catalogue/needs_test.go b/internal/catalogue/needs_test.go new file mode 100644 index 0000000..903d79c --- /dev/null +++ b/internal/catalogue/needs_test.go @@ -0,0 +1,94 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A secret a module needs in order to be itself. +// +// A database has a superuser password, a broker an administrator, a registry an account. None is +// *for* anybody — it is not the credential a consumer is given, and the mechanism that hands +// those out has a consumer in the middle of it. + +func needy() Manifest { + return Manifest{ + Module: "postgres", Version: "1", + Needs: map[string]string{"superuser": "/var/lib/mesh/postgres/superuser"}, + Resources: []map[string]any{ + {"id": "store", "type": "container", "name": "mesh-postgres", "image": "postgres@sha256:x"}, + }, + } +} + +func TestAModulesOwnSecretLandsSealed(t *testing.T) { + got, err := Resolve(shelf(needy()), []string{"postgres"}, reachable(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := got.Declaration(Rendering{ + Needed: map[string]map[string]string{"postgres": {"superuser": "c2VhbGVk"}}, + }) + if err != nil { + t.Fatal(err) + } + for _, r := range out { + if r["path"] != "/var/lib/mesh/postgres/superuser" { + continue + } + if r["sealed"] != "c2VhbGVk" { + t.Fatalf("got %v", r) + } + if r["content"] != nil { + t.Fatal("a module's own secret was written in the clear") + } + return + } + t.Fatalf("no secret was written: %v", out) +} + +func TestADeclaredNeedThatWasNotMadeIsRefused(t *testing.T) { + // Skipping it would start a database with no password it knows, which fails to authenticate + // three layers from the machine reporting it. + got, _ := Resolve(shelf(needy()), []string{"postgres"}, reachable(), World{}) + _, err := got.Declaration(Rendering{}) + if err == nil { + t.Fatal("a module needing a secret was declared without one") + } + if !strings.Contains(err.Error(), "superuser") { + t.Fatalf("the refusal does not name what is missing: %v", err) + } +} + +func TestANeedIsAnAbsolutePath(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"postgres","version":"1", + "needs":{"superuser":"superuser.txt"}}`)) + if err == nil { + t.Fatal("a relative path was accepted") + } + if !strings.Contains(err.Error(), "absolute path") { + t.Fatalf("unhelpful refusal: %v", err) + } +} + +func TestAModuleMayNeedSeveralThings(t *testing.T) { + // A password and a token, say. Telling them apart is the module's business, not the mesh's. + m := needy() + m.Needs["replication"] = "/var/lib/mesh/postgres/replication" + got, _ := Resolve(shelf(m), []string{"postgres"}, reachable(), World{}) + out, err := got.Declaration(Rendering{Needed: map[string]map[string]string{ + "postgres": {"superuser": "b25l", "replication": "dHdv"}, + }}) + if err != nil { + t.Fatal(err) + } + seen := map[string]string{} + for _, r := range out { + if path, ok := r["path"].(string); ok && strings.Contains(path, "/var/lib/mesh/postgres/") { + seen[path], _ = r["sealed"].(string) + } + } + if len(seen) != 2 || seen["/var/lib/mesh/postgres/superuser"] == seen["/var/lib/mesh/postgres/replication"] { + t.Fatalf("two needs did not land as two secrets: %v", seen) + } +} diff --git a/internal/inventory/migrations/0012-a-modules-own-secret.sql b/internal/inventory/migrations/0012-a-modules-own-secret.sql new file mode 100644 index 0000000..46e7d9b --- /dev/null +++ b/internal/inventory/migrations/0012-a-modules-own-secret.sql @@ -0,0 +1,29 @@ +-- A secret a module needs in order to be itself. +-- +-- A database has a superuser password, a broker an administrator, a registry an account. None of +-- them is *for* anybody -- they are not the credential a consumer is given, and the table holding +-- those has a consumer in its key. +-- +-- **One per node**, so a module running on three machines has three passwords. A manifest that +-- carried one instead would put the same secret on every machine that ever runs the module, in a +-- file anybody can read, for ever. +-- +-- Sealed to the node before it is written, like everything else here: what is stored is unusable +-- by whoever holds it, the mesh included. + +create table module_secret ( + node uuid not null references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + -- The module's own word for it. Two secrets in one module are ordinary -- a password and a + -- token, say -- and telling them apart is the module's business, not the mesh's. + name text not null, + + sealed text not null, + -- Which key it was sealed to, so a node that regenerated its key can be told what it can no + -- longer open rather than discovering it as a service that will not start. + node_key text not null, + + made_at timestamptz not null default now(), + + primary key (node, module, name) +); diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 3aaf307..2cd2b51 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "fmt" "github.com/novox/mesh-control/internal/secrets" ) @@ -133,3 +134,49 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret, } return out, rows.Err() } + +// SecretForModule is a secret a module needs in order to be itself, on one machine. +// +// Not the credential a consumer is given: a superuser password is not *for* anybody. Made once +// and kept, because regenerating it on every declaration would change the password a running +// database has already been started with — and remade when the node's sealing key changes, for +// the same reason as everything else sealed here. +func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) { + key, err := i.SealingKeyOf(ctx, node) + if err != nil { + return "", err + } + if key == "" { + return "", fmt.Errorf( + "%s needs a secret and %s has no sealing key, so nothing can be sealed to it", + module, node) + } + record, err := i.NodeByName(ctx, node) + if err != nil { + return "", err + } + + var sealed, against string + err = i.store.Pool().QueryRow(ctx, + `select sealed, node_key from module_secret where node = $1 and module = $2 and name = $3`, + record.ID, module, name).Scan(&sealed, &against) + if err == nil && against == key { + return sealed, nil + } + + made, err := secrets.Make(key, key) + if err != nil { + return "", err + } + // Sealed once, to one recipient. Make seals to two ends because a provision has two; here + // both are the same machine, and only one copy is kept. + if _, err := i.store.Pool().Exec(ctx, + `insert into module_secret (node, module, name, sealed, node_key) + values ($1, $2, $3, $4, $5) + on conflict (node, module, name) do update set + sealed = excluded.sealed, node_key = excluded.node_key, made_at = now()`, + record.ID, module, name, made.ForConsumer, key); err != nil { + return "", err + } + return made.ForConsumer, nil +} diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go index 9a70661..dec409e 100644 --- a/internal/inventory/secrets_test.go +++ b/internal/inventory/secrets_test.go @@ -5,6 +5,7 @@ import ( "crypto/ecdh" "crypto/rand" "encoding/base64" + "github.com/novox/mesh-control/internal/catalogue" "strings" "testing" @@ -230,3 +231,97 @@ func TestSecretsGoWhenANodeLeaves(t *testing.T) { t.Fatalf("%d credential(s) outlived the machine they were for", left) } } + +func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) { + // A module running on three machines has three passwords. One in the manifest instead would + // put the same secret on every machine that ever runs it, in a file anybody can read. + inv, ctx := twoNodesWithKeys(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, + Source{}); err != nil { + t.Fatal(err) + } + + here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + if here == there { + t.Fatal("two machines were given the same secret") + } + + // Made once and kept, or a running database would be handed a password it was not started + // with on the next declaration. + again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + if again != here { + t.Fatal("asking twice made a second secret") + } +} + +func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, + Source{}); err != nil { + t.Fatal(err) + } + one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication") + if err != nil { + t.Fatal(err) + } + if one == two { + t.Fatal("two names gave one secret") + } +} + +func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) { + // The node generated a new sealing key and can no longer open what was sealed to the old one. + inv, ctx := twoNodesWithKeys(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, + Source{}); err != nil { + t.Fatal(err) + } + before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + node, err := inv.NodeByName(ctx, "consumer") + if err != nil { + t.Fatal(err) + } + fresh, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil { + t.Fatal(err) + } + after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + if after == before { + t.Fatal("a machine was handed a secret sealed to a key it no longer has") + } +} + +func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + if _, err := inv.AddNode(ctx, "bare"); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, + Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil { + t.Fatal("a secret was made for a machine that cannot open one") + } +} diff --git a/postgres-provisioner b/postgres-provisioner index a765480..99c7af1 100755 Binary files a/postgres-provisioner and b/postgres-provisioner differ