diff --git a/Makefile b/Makefile index 2134731..bb6bc6f 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,17 @@ objectstore-image: @echo @docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' +# The cache's provisioner, for the same reason as the database's: an ACL user is not a file, +# and the mesh cannot make one -- it discarded the credential it would have to use. +REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION) +REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development + +redis-provisioner-image: + docker build -f examples/redis-provisioner/Dockerfile \ + -t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) . + @echo + @docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' + # The proxy that turns a route grant into traffic reaching a workload. PROXY_IMAGE ?= mesh-route-proxy:$(VERSION) PROXY_DEV_TAG ?= mesh-route-proxy:development diff --git a/examples/modules/grafana.json b/examples/modules/grafana.json new file mode 100644 index 0000000..5dcbbd5 --- /dev/null +++ b/examples/modules/grafana.json @@ -0,0 +1,55 @@ +{ + "module": "grafana", + "version": "1", + "own-secrets": { + "admin": "/var/lib/grafana-module/admin.secret" + }, + "capabilities": [ + "container-runtime" + ], + "listens": [ + { + "port": 3000, + "protocol": "tcp", + "from": "mesh", + "why": "the dashboards. Also 3000 inside, like the forge - which is the mesh's port assignment earning its keep" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/grafana-module", + "mode": "0700" + }, + { + "id": "server-env", + "type": "file", + "path": "/var/lib/grafana-module/server.env", + "mode": "0600", + "content": "GF_SECURITY_ADMIN_PASSWORD=${secret:admin}\n" + }, + { + "id": "data", + "type": "directory", + "path": "/services/grafana/data", + "mode": "0700", + "owner": "472:472" + }, + { + "id": "server", + "type": "container", + "name": "grafana", + "image": "grafana/grafana@sha256:f772d434e8fab0049deb2b1b30abd43342bcfca1537614aa8d36080232cf4283", + "env-file": [ + "/var/lib/grafana-module/server.env" + ], + "ports": [ + "3000" + ], + "volumes": [ + "/services/grafana/data:/var/lib/grafana" + ] + } + ] +} diff --git a/examples/modules/redis.json b/examples/modules/redis.json new file mode 100644 index 0000000..1659609 --- /dev/null +++ b/examples/modules/redis.json @@ -0,0 +1,105 @@ +{ + "module": "redis", + "version": "1", + "provides": [ + { + "name": "redis-cache", + "scope": "mesh" + } + ], + "capabilities": [ + "container-runtime" + ], + "serves": { + "redis-cache": {} + }, + "receives": { + "redis-cache": "/var/lib/redis-module/grants/mesh.json" + }, + "grants": { + "redis-cache": "/var/lib/redis-module/grants" + }, + "own-secrets": { + "default": "/var/lib/redis-module/default.secret" + }, + "listens": [ + { + "port": 6379, + "protocol": "tcp", + "from": "mesh", + "why": "modules on any machine that were granted a cache" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/redis-module", + "mode": "0700" + }, + { + "id": "grants-dir", + "type": "directory", + "path": "/var/lib/redis-module/grants", + "mode": "0700" + }, + { + "id": "data", + "type": "directory", + "path": "/services/redis/data", + "mode": "0700" + }, + { + "id": "server-conf", + "type": "file", + "path": "/var/lib/redis-module/redis.conf", + "mode": "0600", + "content": "requirepass ${secret:default}\naclfile /data/users.acl\nappendonly yes\ndir /data\n" + }, + { + "id": "acl-seed", + "type": "file", + "path": "/services/redis/data/users.acl", + "mode": "0600", + "content": "" + }, + { + "id": "net", + "type": "network", + "name": "redis" + }, + { + "id": "server", + "type": "container", + "name": "redis", + "image": "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf", + "network": "redis", + "ports": [ + "6379" + ], + "volumes": [ + "/services/redis/data:/data", + "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro" + ], + "args": [ + "/etc/redis/redis.conf" + ] + }, + { + "id": "provisioner", + "type": "container", + "name": "mesh-provision-redis", + "image": "mesh-provision-redis@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "network": "redis", + "env": { + "GRANTS": "/var/lib/redis-module/grants", + "MESH_PROVISION_REDIS": "redis:6379", + "MESH_PROVISION_PASSWORD_FILE": "/run/secrets/default" + }, + "volumes": [ + "/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro", + "/var/lib/redis-module/default.secret:/run/secrets/default:ro" + ] + } + ] +} diff --git a/examples/modules/registry.json b/examples/modules/registry.json new file mode 100644 index 0000000..6fe2ba0 --- /dev/null +++ b/examples/modules/registry.json @@ -0,0 +1,52 @@ +{ + "module": "registry", + "version": "1", + "provides": [ + { + "name": "artifact-store", + "scope": "mesh" + } + ], + "claims": [ + { + "name": "the-artifact-store", + "scope": "node" + } + ], + "capabilities": [ + "container-runtime" + ], + "serves": { + "artifact-store": { + "port": 5000 + } + }, + "listens": [ + { + "port": 5000, + "protocol": "tcp", + "from": "mesh", + "why": "every machine pulls images and artifacts from here" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/mesh/registry", + "mode": "0700" + }, + { + "id": "store", + "type": "container", + "name": "mesh-registry", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "ports": [ + "5000:5000" + ], + "volumes": [ + "mesh-registry-data:/var/lib/registry" + ] + } + ] +} diff --git a/examples/modules/umami.json b/examples/modules/umami.json new file mode 100644 index 0000000..aa341e7 --- /dev/null +++ b/examples/modules/umami.json @@ -0,0 +1,59 @@ +{ + "module": "umami", + "version": "1", + "requires": [ + "postgres-database" + ], + "contributes": { + "postgres-database": { + "name": "umami" + } + }, + "binds": { + "postgres-database": "/var/lib/umami/database.json" + }, + "secrets": { + "postgres-database": "/var/lib/umami/database.secret" + }, + "own-secrets": { + "app-secret": "/var/lib/umami/app.secret" + }, + "capabilities": [ + "container-runtime" + ], + "listens": [ + { + "port": 3000, + "protocol": "tcp", + "from": "mesh", + "why": "the analytics pages and the collection endpoint" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/umami", + "mode": "0700" + }, + { + "id": "server-env", + "type": "file", + "path": "/var/lib/umami/server.env", + "mode": "0600", + "content": "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/umami\nAPP_SECRET=${secret:app-secret}\n" + }, + { + "id": "server", + "type": "container", + "name": "umami", + "image": "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a", + "env-file": [ + "/var/lib/umami/server.env" + ], + "ports": [ + "3000" + ] + } + ] +} diff --git a/examples/redis-provisioner/Dockerfile b/examples/redis-provisioner/Dockerfile new file mode 100644 index 0000000..7a3b678 --- /dev/null +++ b/examples/redis-provisioner/Dockerfile @@ -0,0 +1,17 @@ +# The cache provisioner, as a module ships one. +# +# FROM scratch, like the postgres one and unlike the bucket one: it speaks the store's own wire +# protocol directly and needs no client in the image. +FROM golang:1.25-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \ + -o /mesh-provision-redis ./examples/redis-provisioner + +FROM scratch +COPY --from=build /mesh-provision-redis /mesh-provision-redis +# Watching by default, because that is what makes it a module: an ordinary long-running service +# the host supervises, rather than something invoked after every declaration. +ENTRYPOINT ["/mesh-provision-redis", "--watch"] diff --git a/examples/redis-provisioner/main.go b/examples/redis-provisioner/main.go new file mode 100644 index 0000000..371f920 --- /dev/null +++ b/examples/redis-provisioner/main.go @@ -0,0 +1,360 @@ +// A provisioner for Redis, in the form the mesh expects one. +// +// The mesh generated a password, sealed it to the machine that must accept it, and discarded the +// plaintext — so it cannot tell Redis to start accepting it. Something on that machine reads what +// the host wrote and makes it true. This is that something, for the third provision after a +// database and a bucket: a cache. +// +// **It is an example, not part of the control plane** — the same standing as the postgres one, +// whose contract this mirrors line for line: +// +// $GRANTS/mesh.json every consumer, what it asked for, and where its credential is +// $GRANTS/.secret one consumer's password, alone in the file +// +// What a consumer is given is an ACL user scoped to a key prefix. Redis has no databases to hand +// out — SELECT-numbered ones are deprecated in clusters and shared in spirit — so the unit of +// tenancy is the keyspace pattern: a consumer contributing `prefix: photos` gets a user that can +// touch `photos:*` and nothing else. Administration and the dangerous category stay withheld; +// nothing a tenant is granted can flush the store or read another tenant's keys. +// +// Redis is spoken to in RESP directly, over one connection, with no client library. Five commands +// are needed — AUTH, PING, ACL SETUSER, ACL USERS, ACL DELUSER, ACL SAVE — and a dependency large +// enough to hide what they do would be most of this program's size. +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net" + "os" + "os/signal" + "path/filepath" + "sort" + "strings" + "syscall" + "time" +) + +// mark is what this provisioner names the users it owns, so it never removes one a person made by +// hand — the mesh's rule about origins, one level down (novox/hq 04-ISSUES/010). +const mark = "mesh_" + +type contribution struct { + From string `json:"from"` + Node string `json:"node"` + // As is what to call the user this consumer will authenticate as. Given by the mesh, never + // invented here (novox/hq 04-ISSUES/023): the consumer has to present it, so both ends must + // hold the same derivation. + As string `json:"as"` + Secret string `json:"secret"` + Values map[string]any `json:"values"` +} + +type manifest struct { + Requirement string `json:"requirement"` + Given []contribution `json:"given"` +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if len(os.Args) > 1 && os.Args[1] == "--watch" { + if err := watch(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-provision-redis: %v\n", err) + os.Exit(1) + } + return + } + if err := run(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-provision-redis: %v\n", err) + os.Exit(1) + } +} + +// watch reconciles now, and again whenever what the mesh delivered changes — by content, not by +// modification time, for the reasons the postgres provisioner records. +func watch(ctx context.Context) error { + const every = 10 * time.Second + var last string + for { + state, err := given() + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err) + case state != last: + if err := run(ctx); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } else { + last = state + } + } + select { + case <-ctx.Done(): + return nil + case <-time.After(every): + } + } +} + +// given digests everything delivered, so a change of any of it is one comparison and no secret is +// held beyond its hash. +func given() (string, error) { + entries, err := os.ReadDir(grantsDir()) + if err != nil { + return "", err + } + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + sort.Strings(names) + sum := sha256.New() + for _, name := range names { + body, err := os.ReadFile(filepath.Join(grantsDir(), name)) + if err != nil { + return "", err + } + fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body)) + } + return hex.EncodeToString(sum.Sum(nil)), nil +} + +func grantsDir() string { + if grants := os.Getenv("GRANTS"); grants != "" { + return grants + } + return "/var/lib/redis-module/grants" +} + +func run(ctx context.Context) error { + raw, err := os.ReadFile(filepath.Join(grantsDir(), "mesh.json")) + if err != nil { + if os.IsNotExist(err) { + fmt.Printf("nothing has been granted to this machine\n") + return nil + } + return err + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + return fmt.Errorf("the manifest at %s is not readable: %w", grantsDir(), err) + } + + r, err := connect(ctx) + if err != nil { + return err + } + defer r.Close() + + // Reconciling, not applying a change: the same state from wherever it starts. + wanted := map[string]bool{} + for _, c := range m.Given { + if c.Node == "" { + continue + } + prefix, _ := c.Values["prefix"].(string) + if prefix == "" { + return fmt.Errorf("%s asked for a cache and did not say its key prefix", c.From) + } + if err := usablePrefix(prefix); err != nil { + return fmt.Errorf("%s: %w", c.From, err) + } + password, err := os.ReadFile(c.Secret) + if err != nil { + return fmt.Errorf("%s's credential should be at %s and is not there", c.Node, c.Secret) + } + user := c.As + if user == "" { + return fmt.Errorf("%s on %s was granted a cache and the mesh did not say what to "+ + "call its user, so there is no name both ends would agree on", c.From, c.Node) + } + if !strings.HasPrefix(user, mark) { + return fmt.Errorf("%s on %s is to be called %q, which does not begin with %q — "+ + "withdrawal finds this provisioner's work by that prefix, so it would never let "+ + "this one go", c.From, c.Node, user, mark) + } + wanted[user] = true + + // One SETUSER, from reset: the whole grant every time, so a rotation replaces the + // password and a changed prefix replaces the keyspace, with no residue of what was. + if _, err := r.do("ACL", "SETUSER", user, "reset", "on", + ">"+strings.TrimSpace(string(password)), + "~"+prefix+":*", "&"+prefix+":*", + "+@all", "-@admin", "-@dangerous"); err != nil { + return fmt.Errorf("granting %s: %w", user, err) + } + fmt.Printf("granted %s the keyspace %s:*\n", user, prefix) + } + + // And every user this provisioner made that nobody asks for any more — the half usually + // missing, without which a departed consumer keeps a working login for ever. + users, err := r.strings("ACL", "USERS") + if err != nil { + return err + } + for _, user := range users { + if !strings.HasPrefix(user, mark) || wanted[user] { + continue + } + if _, err := r.do("ACL", "DELUSER", user); err != nil { + return fmt.Errorf("revoking %s: %w", user, err) + } + fmt.Printf("revoked %s\n", user) + } + + // Persisted, or the next restart forgets every grant. The server runs with an aclfile for + // exactly this; a server without one refuses the save, and saying so beats a cache that + // silently loses its tenants on restart. + if _, err := r.do("ACL", "SAVE"); err != nil { + return fmt.Errorf("the grants were applied and could not be persisted — a restart would "+ + "forget them: %w", err) + } + return nil +} + +// resp is the five commands this needs, spoken directly. +type resp struct { + conn net.Conn + in *bufio.Reader +} + +func connect(ctx context.Context) (*resp, error) { + where := os.Getenv("MESH_PROVISION_REDIS") + if where == "" { + where = "127.0.0.1:6379" + } + var d net.Dialer + conn, err := d.DialContext(ctx, "tcp", where) + if err != nil { + return nil, err + } + r := &resp{conn: conn, in: bufio.NewReader(conn)} + + file := os.Getenv("MESH_PROVISION_PASSWORD_FILE") + if file == "" { + return nil, fmt.Errorf("MESH_PROVISION_PASSWORD_FILE is not set, and an unauthenticated " + + "provisioner would mean an unauthenticated store") + } + password, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if _, err := r.do("AUTH", strings.TrimSpace(string(password))); err != nil { + return nil, fmt.Errorf("the store did not accept the password the mesh sealed here: %w", err) + } + if _, err := r.do("PING"); err != nil { + return nil, err + } + return r, nil +} + +func (r *resp) Close() { _ = r.conn.Close() } + +// do sends one command and returns the reply, flattened to a string for the simple kinds. +func (r *resp) do(args ...string) (any, error) { + var out strings.Builder + fmt.Fprintf(&out, "*%d\r\n", len(args)) + for _, a := range args { + fmt.Fprintf(&out, "$%d\r\n%s\r\n", len(a), a) + } + if _, err := r.conn.Write([]byte(out.String())); err != nil { + return nil, err + } + return r.read() +} + +// strings is do, for the replies that are arrays of bulk strings. +func (r *resp) strings(args ...string) ([]string, error) { + reply, err := r.do(args...) + if err != nil { + return nil, err + } + items, ok := reply.([]any) + if !ok { + return nil, fmt.Errorf("expected an array and the store said %v", reply) + } + var out []string + for _, item := range items { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + return out, nil +} + +func (r *resp) read() (any, error) { + line, err := r.in.ReadString('\n') + if err != nil { + return nil, err + } + line = strings.TrimRight(line, "\r\n") + if line == "" { + return nil, fmt.Errorf("the store sent an empty reply") + } + body := line[1:] + switch line[0] { + case '+', ':': + return body, nil + case '-': + // The store's own words, verbatim: it says exactly what it refused and why. + return nil, fmt.Errorf("%s", body) + case '$': + if body == "-1" { + return nil, nil + } + var n int + fmt.Sscanf(body, "%d", &n) + buf := make([]byte, n+2) + if _, err := readFull(r.in, buf); err != nil { + return nil, err + } + return string(buf[:n]), nil + case '*': + var n int + fmt.Sscanf(body, "%d", &n) + items := make([]any, 0, n) + for range n { + item, err := r.read() + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil + } + return nil, fmt.Errorf("the store began a reply with %q, which this does not speak", line[0]) +} + +func readFull(in *bufio.Reader, buf []byte) (int, error) { + total := 0 + for total < len(buf) { + n, err := in.Read(buf[total:]) + if err != nil { + return total, err + } + total += n + } + return total, nil +} + +// usablePrefix refuses a prefix that would grant more keyspace than anyone read in the manifest. +// +// The grant is written into an ACL pattern, so a prefix carrying pattern characters stops meaning +// itself: `pho*` granted as `pho*:*` matches every tenant whose name starts with pho. The grant +// must mean what it says, so anything Redis would read as a pattern is refused rather than +// escaped — escaping would create a second naming scheme the mesh does not know about. +func usablePrefix(prefix string) error { + if prefix == "" { + return fmt.Errorf("the key prefix is empty, which would grant the whole keyspace") + } + if strings.ContainsAny(prefix, " \t\n\r*?[]^{}") { + return fmt.Errorf("the key prefix %q contains characters Redis reads as a pattern, so "+ + "the grant would match more than it names", prefix) + } + return nil +} diff --git a/examples/redis-provisioner/naming_test.go b/examples/redis-provisioner/naming_test.go new file mode 100644 index 0000000..bd469d9 --- /dev/null +++ b/examples/redis-provisioner/naming_test.go @@ -0,0 +1,18 @@ +package main + +import "testing" + +// The grant must mean what it says: a prefix Redis would read as a pattern grants keyspace nobody +// saw in the manifest. +func TestAPrefixThatIsAPatternIsRefused(t *testing.T) { + for _, bad := range []string{"", "pho*", "a?b", "x[yz]", "a b", "brace{s}"} { + if err := usablePrefix(bad); err == nil { + t.Errorf("%q was accepted, and would match more keyspace than it names", bad) + } + } + for _, fine := range []string{"photos", "mesh_laptop_realapp", "a-b.c_d"} { + if err := usablePrefix(fine); err != nil { + t.Errorf("%q was refused and names exactly itself: %v", fine, err) + } + } +} diff --git a/redis-provisioner b/redis-provisioner new file mode 100755 index 0000000..96b9ea0 Binary files /dev/null and b/redis-provisioner differ