mesh-controller/examples/objectstore-provisioner/main.go

// A provisioner for buckets, in the form the mesh expects one.
//
// The same contract as `examples/postgres-provisioner`, against a different kind of thing — and
// that is the point of it existing. The mesh generated a secret, sealed it to the machine that
// must accept it, and discarded the plaintext, so it cannot tell an object store to start
// accepting it. Something on that machine reads what the host wrote and makes it true.
//
// **It is an example, not part of the control plane** (novox/hq ADR 0001). A real one ships with
// the module that ships the object store. What lives here is the contract, written as something
// that runs so it can be read rather than described.
//
// What it is given, both written by the host from an ordinary declaration:
//
//	$GRANTS/mesh.json          every consumer, what it asked for, and where its credential is
//	$GRANTS/<node>.secret      one consumer's secret key, alone in the file
//
// **Why it drives the vendor's own command line rather than an SDK.** MinIO's admin API encrypts
// its request bodies, which is why a separate admin library exists; pulling that in would add a
// system-metrics dependency tree to a repository that has none, to create a user. A provisioner
// is allowed to know how to operate the thing it provisions — that is the whole of its job — and
// this way the example stays about the contract instead of about somebody's request signing.
package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"
)

// mark is what this provisioner names the access keys it owns.
//
// So it never removes one a person made by hand — the mesh's own rule about origins, one level
// down (novox/hq 04-ISSUES/010). A provisioner that deleted every user it did not recognise would
// be one nobody could safely run against a store that predates it.
const mark = "mesh_"

// alias is the name `mc` keeps its connection under. Local to this process's config directory.
const alias = "store"

// contribution is one consumer, as the mesh described it.
type contribution struct {
	From string `json:"from"`
	Node string `json:"node"`
	// As is what to call the login this consumer will use.
	//
	// **Given, not invented** (novox/hq 04-ISSUES/023). This provisioner used to make the name
	// itself, which worked and meant the consumer — the one thing that has to present it — could
	// not learn it. The mesh derives it once and sends it to both ends.
	As     string         `json:"as"`
	Secret string         `json:"secret"`
	Values map[string]any `json:"values"`
}

type manifest struct {
	Requirement string         `json:"requirement"`
	Given       []contribution `json:"given"`
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()

	if len(os.Args) > 1 && os.Args[1] == "--watch" {
		if err := watch(ctx); err != nil {
			fmt.Fprintf(os.Stderr, "mesh-provision-objectstore: %v\n", err)
			os.Exit(1)
		}
		return
	}
	if err := run(ctx); err != nil {
		fmt.Fprintf(os.Stderr, "mesh-provision-objectstore: %v\n", err)
		os.Exit(1)
	}
}

// watch reconciles now, and again whenever what the mesh delivered changes.
//
// By polling rather than watching the filesystem: the host writes atomically, so the file is
// replaced rather than modified, and an inotify watch on the path stops seeing anything after the
// first replacement — a watcher that silently stops working.
func watch(ctx context.Context) error {
	const every = 10 * time.Second
	var last string

	for {
		state, err := given()
		switch {
		case err != nil:
			// Said and retried. A provisioner that exits because the mesh has not written
			// anything yet has to be restarted by hand after the first push.
			fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err)
		case state != last:
			if err := run(ctx); err != nil {
				// Reported and retried. The usual reason is that the store has not finished
				// starting, and giving up would mean a module that works only when two
				// containers happen to come up in the right order.
				fmt.Fprintf(os.Stderr, "%v\n", err)
			} else {
				last = state
			}
		}

		select {
		case <-ctx.Done():
			return nil
		case <-time.After(every):
		}
	}
}

// grantsDir is where the host was told to write what this provisioner is given.
func grantsDir() string {
	if d := os.Getenv("GRANTS"); d != "" {
		return d
	}
	return "/var/lib/objectstore/grants"
}

// given is everything the mesh has delivered, as one string, so a change to any of it is one
// comparison.
//
// Credentials are included by their **digest**, never their content: this is compared, held in
// memory for the life of the process, and printed on failure, and a secret belongs in none of
// those when a hash answers the same question.
func given() (string, error) {
	entries, err := os.ReadDir(grantsDir())
	if err != nil {
		return "", err
	}
	var names []string
	for _, e := range entries {
		names = append(names, e.Name())
	}
	sort.Strings(names)

	sum := sha256.New()
	for _, name := range names {
		body, err := os.ReadFile(filepath.Join(grantsDir(), name))
		if err != nil {
			return "", err
		}
		fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body))
	}
	return hex.EncodeToString(sum.Sum(nil)), nil
}

func run(ctx context.Context) error {
	raw, err := os.ReadFile(filepath.Join(grantsDir(), "mesh.json"))
	if err != nil {
		if os.IsNotExist(err) {
			// Nothing has been granted here. Not a failure: a provider with no consumers is an
			// ordinary state, and one this must be able to reach from any other.
			fmt.Printf("nothing has been granted to this machine\n")
			return nil
		}
		return err
	}
	var m manifest
	if err := json.Unmarshal(raw, &m); err != nil {
		return fmt.Errorf("the manifest at %s is not readable: %w", grantsDir(), err)
	}

	if err := connect(ctx); err != nil {
		return err
	}

	// **Reconciling, not applying a change.** It runs after every declaration and is never told
	// what changed, so it must reach the same state from wherever it starts.
	wanted := map[string]bool{}
	for _, c := range sorted(m.Given) {
		if c.Node == "" {
			continue
		}
		bucket, _ := c.Values["bucket"].(string)
		if bucket == "" {
			return fmt.Errorf("%s asked for a bucket and did not name it", c.Node)
		}
		if err := usableBucketName(bucket); err != nil {
			return fmt.Errorf("%s asked for a bucket named %q: %w", c.Node, bucket, err)
		}
		secret, err := os.ReadFile(c.Secret)
		if err != nil {
			// The manifest says there is a credential and the host has not written it. Refused
			// rather than creating a user with no secret — a login nothing can use, which
			// nothing would report until something tried to connect.
			return fmt.Errorf("%s's credential should be at %s and is not there", c.Node, c.Secret)
		}

		// One access key per consumer, and a consumer is a module on a machine (novox/hq
		// 04-ISSUES/022) — otherwise every service on a node shares one key, and the policy that
		// confines each to its own bucket confines none of them.
		//
		// The name comes from the mesh (04-ISSUES/023), so the consumer knows what it will be.
		key := c.As
		if key == "" {
			return fmt.Errorf(
				"%s on %s was granted a bucket and the mesh did not say what to call its key, "+
					"so there is no name both ends would agree on", c.From, c.Node)
		}
		if !strings.HasPrefix(key, mark) {
			return fmt.Errorf(
				"%s on %s is to be called %q, which does not begin with %q — this provisioner "+
					"removes what it made by that prefix, so it would never let this one go",
				c.From, c.Node, key, mark)
		}
		wanted[key] = true
		if err := ensureBucket(ctx, bucket); err != nil {
			return err
		}
		if err := ensureUser(ctx, key, strings.TrimSpace(string(secret))); err != nil {
			return err
		}
		if err := ensureOnlyThatBucket(ctx, key, bucket); err != nil {
			return err
		}
	}

	// And everything this provisioner made that nobody asks for any more. **The half usually
	// missing**: a consumer that goes away otherwise keeps a working key for ever, and nothing
	// says so.
	return revokeOrphans(ctx, wanted)
}

// sorted puts the consumers in a stable order, so two runs over the same input do the same things
// in the same sequence and a log can be compared against another.
func sorted(given []contribution) []contribution {
	out := append([]contribution(nil), given...)
	sort.Slice(out, func(i, j int) bool {
		if out[i].Node != out[j].Node {
			return out[i].Node < out[j].Node
		}
		return out[i].From < out[j].From
	})
	return out
}

// usableBucketName refuses a name the store would refuse, but says so in terms of the module that
// asked rather than in terms of a REST error nobody can trace back.
//
// Deliberately narrower than the store's own rule: this rejects what is ambiguous as well as what
// is invalid, because a bucket named `Photos` that arrives as `photos` is a module that works
// until somebody looks.
func usableBucketName(name string) error {
	if len(name) < 3 || len(name) > 63 {
		return fmt.Errorf("a bucket name is 3 to 63 characters")
	}
	for _, r := range name {
		if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' {
			return fmt.Errorf("only lower-case letters, digits and dashes are usable, and %q is not", r)
		}
	}
	if name[0] == '-' || name[len(name)-1] == '-' {
		return fmt.Errorf("it may not begin or end with a dash")
	}
	return nil
}

// onlyThatBucket is the policy a consumer gets: its own bucket, and nothing else in the store.
//
// **Written per consumer rather than shared.** A single policy naming every bucket would grow a
// line each time somebody asks for one, and every existing consumer would silently gain access to
// the new one.
func onlyThatBucket(bucket string) string {
	return fmt.Sprintf(`{
  "Version": "2012-10-17",
  "Statement": [
    {"Effect": "Allow",
     "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
     "Resource": ["arn:aws:s3:::%s"]},
    {"Effect": "Allow",
     "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
     "Resource": ["arn:aws:s3:::%s/*"]}
  ]
}`, bucket, bucket)
}

func mc(ctx context.Context, args ...string) (string, error) {
	// Its own configuration directory, so this never reads or writes whatever a person running
	// `mc` on this machine has set up.
	cmd := exec.CommandContext(ctx, "mc", append([]string{"--config-dir", "/tmp/mesh-mc"}, args...)...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		// The command is named without its arguments: an argument here can be a secret.
		return string(out), fmt.Errorf("mc %s: %w\n%s", args[0], err, strings.TrimSpace(string(out)))
	}
	return string(out), nil
}

// connect points `mc` at the store, using the root credential this provisioner was given.
func connect(ctx context.Context) error {
	endpoint := os.Getenv("MESH_OBJECTSTORE_URL")
	if endpoint == "" {
		endpoint = "http://127.0.0.1:9000"
	}
	user := os.Getenv("MESH_OBJECTSTORE_ROOT_USER")
	file := os.Getenv("MESH_OBJECTSTORE_ROOT_PASSWORD_FILE")
	if user == "" || file == "" {
		// Named together, because a provisioner that starts with half its credential fails later
		// against the store and reads as the store being wrong.
		return fmt.Errorf("MESH_OBJECTSTORE_ROOT_USER and MESH_OBJECTSTORE_ROOT_PASSWORD_FILE " +
			"must both be set: this provisions the store and has to authenticate to it")
	}
	// From a file, never an environment variable: the environment of a process is readable by
	// anything that can see the process, and this one is the store's root.
	password, err := os.ReadFile(file)
	if err != nil {
		return fmt.Errorf("the store's root password should be at %s and is not there", file)
	}
	_, err = mc(ctx, "alias", "set", alias, endpoint, user, strings.TrimSpace(string(password)))
	return err
}

func ensureBucket(ctx context.Context, bucket string) error {
	_, err := mc(ctx, "mb", "--ignore-existing", alias+"/"+bucket)
	return err
}

// ensureUser sets the secret every time rather than only on creation.
//
// The mesh replaces the file when it rotates, and a provisioner that only ever created would
// leave the old key working — a rotation that reports success and changes nothing.
func ensureUser(ctx context.Context, key, secret string) error {
	_, err := mc(ctx, "admin", "user", "add", alias, key, secret)
	return err
}

func ensureOnlyThatBucket(ctx context.Context, key, bucket string) error {
	name := key + "-" + bucket
	path := filepath.Join("/tmp", name+".json")
	if err := os.WriteFile(path, []byte(onlyThatBucket(bucket)), 0o600); err != nil {
		return err
	}
	defer os.Remove(path)

	// Replaced rather than created-if-absent: the bucket a consumer asks for can change, and a
	// policy that was only ever created would keep granting the old one as well.
	_, _ = mc(ctx, "admin", "policy", "remove", alias, name)
	if _, err := mc(ctx, "admin", "policy", "create", alias, name, path); err != nil {
		return err
	}
	_, err := mc(ctx, "admin", "policy", "attach", alias, name, "--user", key)
	if err != nil && strings.Contains(err.Error(), "already") {
		// Attaching a policy that is already attached is the state being asked for.
		return nil
	}
	return err
}

// revokeOrphans removes the keys this provisioner made that nobody asks for any more.
func revokeOrphans(ctx context.Context, wanted map[string]bool) error {
	out, err := mc(ctx, "admin", "user", "list", alias, "--json")
	if err != nil {
		return err
	}
	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
		if line == "" {
			continue
		}
		var u struct {
			AccessKey string `json:"accessKey"`
		}
		if json.Unmarshal([]byte(line), &u) != nil {
			continue
		}
		if !strings.HasPrefix(u.AccessKey, mark) || wanted[u.AccessKey] {
			continue
		}
		if _, err := mc(ctx, "admin", "user", "remove", alias, u.AccessKey); err != nil {
			return err
		}
		fmt.Printf("revoked %s, which nothing asks for any more\n", u.AccessKey)
	}
	return nil
}