// A provisioner, in the form the mesh expects one. // // The mesh generated a password, sealed it to the machine that must accept it, and discarded the // plaintext — so it cannot tell PostgreSQL to start accepting it. Something on that machine reads // what the host wrote and makes it true. This is that something. // // **It is an example, not part of the control plane.** The control plane decides and never // touches a machine; this runs on the machine and touches it. A real one ships with the module // that ships PostgreSQL (novox/hq ADR 0001 — third-party software runs *on* the mesh, not *of* // it). What lives here is the contract, written as something that runs so it can be read rather // than described. // // What it is given, both written by the host from an ordinary declaration: // // $GRANTS/mesh.json every consumer, what it asked for, and where its credential is // $GRANTS/.secret one consumer's password, alone in the file // // Two files because the mesh discarded the value and could not compose a document containing it. package main import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "net/url" "os" "os/signal" "path/filepath" "sort" "strings" "syscall" "time" "github.com/jackc/pgx/v5" ) // mark is what this provisioner names the roles it owns. // // So it never removes one a person made by hand — the mesh's own rule about origins, one level // down (novox/hq 04-ISSUES/010). A provisioner that dropped every role it did not recognise would // be a provisioner nobody could safely run on a database that predates it. const mark = "mesh_" // contribution is one consumer, as the mesh described it. type contribution struct { From string `json:"from"` // Node is empty for a module on this machine, which is asking for something local and is not // this provisioner's business. Node string `json:"node"` Secret string `json:"secret"` Values map[string]any `json:"values"` } type manifest struct { Requirement string `json:"requirement"` Given []contribution `json:"given"` } func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() // Once, or whenever what it was given changes. // // **Watching is what lets this be a module.** Run once, it has to be invoked by something // after every declaration — a timer that runs it when nothing changed, or a unit wired to // restart on a file. Watching, it is an ordinary long-running service, which is a shape the // mesh already delivers and the host already supervises. // // The file it watches is the manifest the mesh writes. A credential changing rewrites the // file beside it and not the manifest, so the manifest is stamped whenever either is written // — which is why this compares content rather than modification time. if len(os.Args) > 1 && os.Args[1] == "--watch" { if err := watch(ctx); err != nil { fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err) os.Exit(1) } return } if err := run(ctx); err != nil { fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err) os.Exit(1) } } // watch reconciles now, and again whenever what the mesh delivered changes. // // By polling rather than by watching the filesystem, because the file is replaced rather than // written in place — the host writes atomically, so an inotify watch on the path stops seeing // anything after the first replacement, which is a watcher that silently stops working. func watch(ctx context.Context) error { const every = 10 * time.Second var last string for { state, err := given() switch { case err != nil: // Said and retried. A provisioner that exits because the mesh has not written // anything yet is one that has to be restarted by hand after the first push. fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err) case state != last: if err := run(ctx); err != nil { // Reported and retried. The usual reason is that the database has not finished // starting, and giving up would mean a module that works only if the two // containers happen to come up in the right order. fmt.Fprintf(os.Stderr, "%v\n", err) } else { last = state } } select { case <-ctx.Done(): return nil case <-time.After(every): } } } // given is everything the mesh has delivered, as one string, so a change of any of it is one // comparison. // // The credentials are included by their **digest**, never their content: this is compared, logged // on nothing, and held in memory for as long as the process runs, and a secret does not belong in // any of that when a hash answers the same question. func given() (string, error) { grants := os.Getenv("GRANTS") if grants == "" { grants = "/var/lib/postgres/grants" } entries, err := os.ReadDir(grants) if err != nil { return "", err } var names []string for _, e := range entries { names = append(names, e.Name()) } sort.Strings(names) sum := sha256.New() for _, name := range names { body, err := os.ReadFile(filepath.Join(grants, name)) if err != nil { return "", err } fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body)) } return hex.EncodeToString(sum.Sum(nil)), nil } func run(ctx context.Context) error { grants := os.Getenv("GRANTS") if grants == "" { grants = "/var/lib/postgres/grants" } raw, err := os.ReadFile(filepath.Join(grants, "mesh.json")) if err != nil { if os.IsNotExist(err) { // Nothing has been granted here. Not a failure: a provider with no consumers is an // ordinary state, and one this must be able to reach from any other. fmt.Printf("nothing has been granted to this machine\n") return nil } return err } var m manifest if err := json.Unmarshal(raw, &m); err != nil { return fmt.Errorf("the manifest at %s is not readable: %w", grants, err) } where, err := connectionString() if err != nil { return err } db, err := pgx.Connect(ctx, where) if err != nil { return err } defer db.Close(ctx) // **Reconciling, not applying a change.** It runs after every declaration and is never told // what changed, so it must reach the same state from wherever it starts. wanted := map[string]bool{} for _, c := range sorted(m.Given) { if c.Node == "" { continue } name, _ := c.Values["name"].(string) if name == "" { return fmt.Errorf("%s asked for a database and did not name it", c.Node) } password, err := os.ReadFile(c.Secret) if err != nil { // The manifest says there is a credential and the host has not written it. Refused // rather than creating a role with no password — a login nothing can use, which // nothing would report until something tried to connect. return fmt.Errorf("%s's credential should be at %s and is not there", c.Node, c.Secret) } // **Named after the module and the machine, not the machine** (novox/hq 04-ISSUES/022). // A node routinely runs several services against one database server, and one role for // all of them means gitea's login opens keycloak's data — created exactly as asked, with // nothing anywhere to say so. It also makes withdrawal impossible: one role cannot be // removed for one consumer while another still holds it. role := mark + c.Node + "_" + c.From wanted[role] = true if err := ensureRole(ctx, db, role, strings.TrimSpace(string(password))); err != nil { return err } if err := ensureDatabase(ctx, db, name, role); err != nil { return err } } // And everything this provisioner made that nobody asks for any more. **The half usually // missing**: a consumer that goes away otherwise keeps a working login for ever and nothing // says so. return revokeOrphans(ctx, db, wanted) } // identifierLimit is where PostgreSQL stops reading a name: NAMEDATALEN - 1. const identifierLimit = 63 // usableRole refuses a role name PostgreSQL would silently shorten. // // **Truncation is a NOTICE, not an error.** A name past the limit is cut to fit and the statement // succeeds, so two consumers whose names agree for the first 63 bytes become one role — which is // the exact fault 022 was about, reappearing at a length nobody would think to test. Refusing is // the only honest answer: the provisioner cannot shorten the name itself without inventing a // second naming scheme that the mesh does not know about, and would then be creating a login the // mesh cannot name. func usableRole(role string) error { if len(role) <= identifierLimit { return nil } return fmt.Errorf( "the role for this consumer would be %q, which is %d bytes and PostgreSQL keeps %d — "+ "it would be shortened silently, and another consumer shortened to the same name "+ "would share the login. Shorten the node or module name", role, len(role), identifierLimit) } func ensureRole(ctx context.Context, db *pgx.Conn, role, password string) error { if err := usableRole(role); err != nil { return err } var exists bool if err := db.QueryRow(ctx, `select true from pg_roles where rolname = $1`, role).Scan(&exists); err != nil && err != pgx.ErrNoRows { return err } // Set every time rather than only on creation. The mesh replaces the file when it rotates, // and a provisioner that only ever created would leave the old password working — a rotation // that reports success and changes nothing. verb := "create" if exists { verb = "alter" } _, err := db.Exec(ctx, fmt.Sprintf("%s role %s with login password %s", verb, quoteName(role), quoteString(password))) if err != nil { return err } if !exists { fmt.Printf("created %s\n", role) } return nil } func ensureDatabase(ctx context.Context, db *pgx.Conn, name, owner string) error { var exists bool if err := db.QueryRow(ctx, `select true from pg_database where datname = $1`, name).Scan(&exists); err != nil && err != pgx.ErrNoRows { return err } if exists { return nil } if _, err := db.Exec(ctx, fmt.Sprintf("create database %s owner %s", quoteName(name), quoteName(owner))); err != nil { return err } fmt.Printf("created database %s owned by %s\n", name, owner) return nil } func revokeOrphans(ctx context.Context, db *pgx.Conn, wanted map[string]bool) error { rows, err := db.Query(ctx, `select rolname from pg_roles where rolname like $1 and rolcanlogin order by rolname`, mark+"%") if err != nil { return err } var found []string for rows.Next() { var role string if err := rows.Scan(&role); err != nil { rows.Close() return err } found = append(found, role) } rows.Close() if err := rows.Err(); err != nil { return err } for _, role := range found { if wanted[role] { continue } // Login removed rather than the role dropped. Dropping fails while the role owns // anything, and a provisioner that failed there would stop reconciling everything else — // so the credential stops working immediately and what it owns is somebody's to decide // about. if _, err := db.Exec(ctx, fmt.Sprintf("alter role %s with nologin", quoteName(role))); err != nil { return err } fmt.Printf("revoked %s — nothing in the mesh asks for it\n", role) } return nil } // sorted puts consumers in a stable order, so two runs do the same work in the same sequence and // the output of one can be compared with another. func sorted(given []contribution) []contribution { out := append([]contribution{}, given...) sort.Slice(out, func(i, j int) bool { if out[i].Node != out[j].Node { return out[i].Node < out[j].Node } return out[i].From < out[j].From }) return out } // quoteName and quoteString exist because PostgreSQL takes no parameters in DDL. // // Both double the quote character, which is the whole of the escaping rule. Worth doing properly // even here: a password is chosen by the mesh and a node name by a person, and "the value happens // to be safe today" is not a property anything should rest on. func quoteName(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } func quoteString(s string) string { return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` } // connectionString is where this provisioner reaches the database it owns. // // **The password comes from a file**, because that is how the mesh delivers one. A module's own // secret — a superuser password here — is sealed to the machine and written by the host; a // provisioner told to take it from an environment variable would need somebody to read the file // and pass it in, which is a person in the middle of the one path that exists so there is not // one. // // It is also the difference between a credential that lives in a file and one that lives in a // process listing: `docker inspect` prints environment, and a superuser password printed by an // ordinary diagnostic is a superuser password in whatever collected that diagnostic. // // MESH_PROVISION_POSTGRES alone still works, for a provisioner somebody runs by hand. func connectionString() (string, error) { where := strings.TrimSpace(os.Getenv("MESH_PROVISION_POSTGRES")) if where == "" { return "", fmt.Errorf( "MESH_PROVISION_POSTGRES is not set, so this provisioner does not know which " + "database it owns") } path := strings.TrimSpace(os.Getenv("MESH_PROVISION_PASSWORD_FILE")) if path == "" { return where, nil } raw, err := os.ReadFile(path) if err != nil { return "", fmt.Errorf( "cannot read the password this provisioner was given at %s: %w", path, err) } password := strings.TrimSpace(string(raw)) if password == "" { // An empty file connects as nobody and is refused by the database, three layers from // here, as an authentication problem with no cause anybody changed. return "", fmt.Errorf("%s is empty, so this provisioner has no password", path) } parsed, err := url.Parse(where) if err != nil { return "", fmt.Errorf("MESH_PROVISION_POSTGRES is not a URL: %w", err) } user := parsed.User.Username() if user == "" { user = "postgres" } parsed.User = url.UserPassword(user, password) return parsed.String(), nil }