package inventory import ( "context" "errors" "fmt" "github.com/jackc/pgx/v5" "github.com/novox/mesh-controller/internal/secrets" ) // Where sealed secrets live. // // The table holds nothing usable — see the migration and internal/secrets for why that is the // design rather than an inconvenience. // Secret is one provision's credential, sealed to each end. type Secret struct { Name string Consumer string // ConsumerModule is which module on that machine it is for. // // **Part of the key, not a label** (novox/hq 04-ISSUES/022). Two modules on one node wanting // the same provision are two consumers, and were one credential until this. ConsumerModule string // Local is the name the credential goes by inside the consumer where it keeps several for one // provision (novox/hq ADR 0094); empty for the ordinary one. Part of the key. Local string Provider string ForConsumer string ForProvider string ConsumerKey string ProviderKey string // Origin is `made` — the mesh generated it — or `accepted` — a person supplied it, for // something outside the mesh, and the mesh cannot make another (novox/hq 04-ISSUES/070). Origin string } // Where a pair credential came from. const ( OriginMade = "made" OriginAccepted = "accepted" ) // SecretFor is the credential one module uses for one provision, making it the first time. // // **Made once and kept**, rather than regenerated whenever it is asked for. A secret that changed // on every declaration would restart both ends on every push and would mean the password a // provider was told to create never matches the one a consumer was given — which is a mesh that // reports success and cannot connect. // // **Remade when either end's sealing key changes.** A node that rejoined generated a new key and // can no longer open what was sealed to the old one, so keeping the blob would deliver something // unreadable for ever. The new secret reaches both ends in the same push, which is the only // moment they can be changed together. func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider, local string) ( Secret, error) { consumerKey, err := i.SealingKeyOf(ctx, consumer) if err != nil { return Secret{}, err } providerKey, err := i.SealingKeyOf(ctx, provider) if err != nil { return Secret{}, err } consumerNode, err := i.NodeByName(ctx, consumer) if err != nil { return Secret{}, err } providerNode, err := i.NodeByName(ctx, provider) if err != nil { return Secret{}, err } var held Secret err = i.store.Pool().QueryRow(ctx, `select for_consumer, for_provider, consumer_key, provider_key, origin from secret where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, name, consumerNode.ID, consumerModule, providerNode.ID, local). Scan(&held.ForConsumer, &held.ForProvider, &held.ConsumerKey, &held.ProviderKey, &held.Origin) if err == nil && held.ConsumerKey == consumerKey && held.ProviderKey == providerKey { held.Name, held.Consumer, held.Provider = name, consumer, provider held.ConsumerModule, held.Local = consumerModule, local return held, nil } if err == nil && held.Origin == OriginAccepted { // A person supplied this, and the mesh does not hold the value: it cannot seal it to the // new key. Refused aloud rather than replaced by something the mesh made up, which would // be delivered, reported as applied, and fail to authenticate somewhere else entirely // (novox/hq 04-ISSUES/070). return Secret{}, fmt.Errorf( "%s's %q credential from %s was accepted from a person, and a sealing key at one end "+ "has changed since. The mesh cannot re-seal a value it does not hold: accept it "+ "again with `secret accept %s %s %s --provider %s%s`", consumerModule, name, provider, consumer, consumerModule, name, provider, localFlag(local)) } // And to the operator, when the mesh has one (novox/hq ADR 0085, amended): the third copy that // makes a vault-provided secret recoverable, and nothing the mesh can open. operator, err := i.OperatorKey(ctx) if err != nil { return Secret{}, err } made, blob, err := secrets.MakeWithOperator(consumerKey, providerKey, operator) if err != nil { return Secret{}, err } forOperator, operatorKey := operatorColumns(operator, blob) _, err = i.store.Pool().Exec(ctx, `insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider, consumer_key, provider_key, operator_sealed, operator_key, local) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) on conflict (name, local, consumer, consumer_module, provider) do update set for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, created_at = now(), operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, name, consumerNode.ID, consumerModule, providerNode.ID, made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey, local) if err != nil { return Secret{}, err } return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule, Local: local, Provider: provider, ForConsumer: made.ForConsumer, ForProvider: made.ForProvider, ConsumerKey: made.ConsumerKey, ProviderKey: made.ProviderKey, Origin: OriginMade}, nil } // AcceptSecretForPair takes a value a person supplied into a pair credential — sealed to the // consumer's node and to the provider's, and to the operator when the mesh has one — where the // mesh would otherwise have made one (novox/hq 04-ISSUES/070, ADR 0092). // // This is the vault's third species: a credential for something outside the mesh, which only a // person can supply. It is the counterpart to AcceptSecretForModule for a module's own secret; // what differs is that both ends of the pair are sealed to, and that the record says `accepted` // so a later read never replaces it with a minted one. The plaintext is discarded here. func (i *Inventory) AcceptSecretForPair(ctx context.Context, name, consumer, consumerModule, provider, local, value string) error { consumerKey, err := i.SealingKeyOf(ctx, consumer) if err != nil { return err } providerKey, err := i.SealingKeyOf(ctx, provider) if err != nil { return err } if consumerKey == "" || providerKey == "" { return fmt.Errorf( "both %s and %s need a sealing key before a credential can be sealed to them — a "+ "node joins to get one", consumer, provider) } consumerNode, err := i.NodeByName(ctx, consumer) if err != nil { return err } providerNode, err := i.NodeByName(ctx, provider) if err != nil { return err } sealed, err := secrets.Accept(value, consumerKey, providerKey) if err != nil { return err } forOperator, operatorKey, err := i.operatorSeal(ctx, value) if err != nil { return err } _, err = i.store.Pool().Exec(ctx, `insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider, consumer_key, provider_key, operator_sealed, operator_key, origin, local) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) on conflict (name, local, consumer, consumer_module, provider) do update set for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, created_at = now(), origin = excluded.origin, operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, name, consumerNode.ID, consumerModule, providerNode.ID, sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey, forOperator, operatorKey, OriginAccepted, local) return err } // RotateSecret discards what was there, so the next declaration carries a new one. // // Only a delete. Nothing reads the old value first, because nothing can — and making the // replacement here rather than on the next read would be a second path to the same act, which is // how two ends come to hold different passwords. // // The new secret then reaches both ends on the same push, together, which is what makes rotation // a single event rather than a fanout with a window where half the mesh holds a dead credential. // // **An accepted credential is not rotated.** The mesh did not make it and cannot make its // replacement; deleting it would have the next read mint one, which is exactly the wrong value // delivered with the mesh insisting it was (novox/hq 04-ISSUES/070). Refused, and the remedy named. func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerModule, provider, local string) error { consumerNode, err := i.NodeByName(ctx, consumer) if err != nil { return err } providerNode, err := i.NodeByName(ctx, provider) if err != nil { return err } var origin string err = i.store.Pool().QueryRow(ctx, `select origin from secret where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, name, consumerNode.ID, consumerModule, providerNode.ID, local).Scan(&origin) if err == nil && origin == OriginAccepted { return fmt.Errorf( "%s's %q credential from %s was accepted from a person, and the mesh cannot make "+ "its replacement. Accept the new value instead: `secret accept %s %s %s "+ "--provider %s%s --from `", consumerModule, name, provider, consumer, consumerModule, name, provider, localFlag(local)) } _, err = i.store.Pool().Exec(ctx, `delete from secret where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, name, consumerNode.ID, consumerModule, providerNode.ID, local) return err } // SecretsFrom is every credential a provider node was issued, so it can be told what to create. func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret, error) { providerNode, err := i.NodeByName(ctx, provider) if err != nil { return nil, err } rows, err := i.store.Pool().Query(ctx, `select s.name, c.name, s.consumer_module, s.local, s.for_provider from secret s join node c on c.id = s.consumer where s.provider = $1 order by s.name, c.name, s.consumer_module, s.local`, providerNode.ID) if err != nil { return nil, err } defer rows.Close() var out []Secret for rows.Next() { s := Secret{Provider: provider} if err := rows.Scan(&s.Name, &s.Consumer, &s.ConsumerModule, &s.Local, &s.ForProvider); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } // SecretForModule is a secret a module needs in order to be itself, on one machine. // // Not the credential a consumer is given: a superuser password is not *for* anybody. Made once // and kept, because regenerating it on every declaration would change the password a running // database has already been started with — and remade when the node's sealing key changes, for // the same reason as everything else sealed here. // ModuleSecretIfIssued is what a module already holds on a node, and nothing if it holds nothing. // // **The read half of SecretForModule**, which mints one when there is none — an insert, and a row // lock, on a path that also serves questions. Composing a declaration to answer *is this machine // running what I would send it* went through the minting version for every module on every node, // so asking wrote to the database and blocked against the machine it was asking about. // // A module with no secret yet has never been sent one, which is the same answer the caller wanted // anyway: this machine is not running what the mesh would send it. func (i *Inventory) ModuleSecretIfIssued( ctx context.Context, node, module, name string, ) (string, bool, error) { key, err := i.SealingKeyOf(ctx, node) if err != nil || key == "" { return "", false, err } record, err := i.NodeByName(ctx, node) if err != nil { return "", false, err } var sealed, against, origin string err = i.store.Pool().QueryRow(ctx, `select sealed, node_key, origin from module_secret where node = $1 and module = $2 and name = $3`, record.ID, module, name).Scan(&sealed, &against, &origin) if errors.Is(err, pgx.ErrNoRows) { return "", false, nil } if err != nil { return "", false, err } // Sealed to a key the node no longer has is not something it holds. Reported as absent rather // than as an error: this is the read, and refusing here would make a question fail for a // condition its writing counterpart is the right place to explain. if against != key { return "", false, nil } return sealed, true, nil } func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) { key, err := i.SealingKeyOf(ctx, node) if err != nil { return "", err } if key == "" { return "", fmt.Errorf( "%s needs a secret and %s has no sealing key, so nothing can be sealed to it", module, node) } record, err := i.NodeByName(ctx, node) if err != nil { return "", err } var sealed, against, origin string err = i.store.Pool().QueryRow(ctx, `select sealed, node_key, origin from module_secret where node = $1 and module = $2 and name = $3`, record.ID, module, name).Scan(&sealed, &against, &origin) if err == nil && against == key { return sealed, nil } if err == nil && origin == "accepted" { // Sealed to a key this node no longer has, and not the mesh's to invent again. Making one // would put 32 random bytes where a working credential was: the machine would apply it, // report success, and whatever reads it would fail to authenticate somewhere else // entirely — with the mesh insisting the secret was delivered, which it was. return "", fmt.Errorf( "%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ "since generated a new sealing key. The mesh cannot make another; issue it again", module, node, name, node) } operator, err := i.OperatorKey(ctx) if err != nil { return "", err } // Sealed once to the machine — Make seals to two ends because a provision has two; here both // are the same machine, and only one copy is kept — and once more to the operator when the // mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from. made, blob, err := secrets.MakeWithOperator(key, key, operator) if err != nil { return "", err } forOperator, operatorKey := operatorColumns(operator, blob) if _, err := i.store.Pool().Exec(ctx, `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) values ($1, $2, $3, $4, $5, 'made', $6, $7) on conflict (node, module, name) do update set sealed = excluded.sealed, node_key = excluded.node_key, origin = excluded.origin, made_at = now(), operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil { return "", err } return made.ForConsumer, nil } // AcceptSecretForModule keeps a value somebody supplied as a module's own secret. // // The counterpart to SecretForModule, which generates one. Some of what a module needs the mesh // cannot invent: a broker account exists because the broker was told about it, and the password is // whatever was agreed with the broker at that moment. The mesh's job is to carry it to the machine // that will use it without being able to read it afterwards. // // Sealed on the way in and the plaintext discarded, exactly as a generated one is — so the only // difference between the two is where the value came from. func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, name, value string) error { key, err := i.SealingKeyOf(ctx, node) if err != nil { return err } if key == "" { return fmt.Errorf( "%s has no sealing key, so nothing can be sealed to it — it joins again to get one", node) } record, err := i.NodeByName(ctx, node) if err != nil { return err } sealed, err := secrets.Accept(value, key, key) if err != nil { return err } // And to the operator, when the mesh has one: a value a person supplied is the one a person // most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended). forOperator, operatorKey, err := i.operatorSeal(ctx, value) if err != nil { return err } _, err = i.store.Pool().Exec(ctx, `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) values ($1, $2, $3, $4, $5, 'accepted', $6, $7) on conflict (node, module, name) do update set sealed = excluded.sealed, node_key = excluded.node_key, origin = excluded.origin, made_at = now(), operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey) return err } // Holder is one end-to-end credential: who gets it and who must create it. type Holder struct { Provision string Consumer string // ConsumerModule is which module on that machine holds it. Part of what identifies a // credential (novox/hq 04-ISSUES/022), so rotating one consumer's does not touch another's. ConsumerModule string // Local is the credential's name inside the consumer where it holds several (ADR 0094). Local string Provider string } // HoldersOf is every pair sharing a credential for one provision. // // **The question rotation has to ask, and the one HAL could not.** There, a provision had a single // shared credential and rotating it updated the provider's row; nothing enumerated who else held // the old one, so three nodes carried dead credentials for two days and the mesh reported success // (novox/hq ADR 0001). Here each pair has its own credential, and this is the list that makes // "every consumer" a set the mesh can name rather than a hope. // // Empty consumer means all of them. func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) ([]Holder, error) { rows, err := i.store.Pool().Query(ctx, `select s.name, c.name, s.consumer_module, s.local, p.name from secret s join node c on c.id = s.consumer join node p on p.id = s.provider where s.name = $1 and ($2 = '' or c.name = $2) order by c.name, s.consumer_module, s.local, p.name`, provision, consumer) if err != nil { return nil, err } defer rows.Close() var out []Holder for rows.Next() { var h Holder if err := rows.Scan(&h.Provision, &h.Consumer, &h.ConsumerModule, &h.Local, &h.Provider); err != nil { return nil, err } out = append(out, h) } return out, rows.Err() } // localFlag is the `--local` a remedy has to name where a credential has a local name. func localFlag(local string) string { if local == "" { return "" } return " --local " + local }