mesh-catalog/modules/zfs/cmd/zfs-tools/keeper.go

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"
)

// The keeper (design 42 zfs-owns-zfs A-37, §18, §20, §26, §27; ADR 0436 §1, §2).
//
// A process's bus account is a user the module makes, never root, so the three root processes (the apply, the run and
// the event follower) declare no `bus`. The process `zfs-keeper`, running as the module's user `zfs-keeper`, holds the
// module's one process bus account and serves a unix socket in the module's state directory that only root may open
// (the peer's uid is checked on every connection). Its calls are a closed list: emit an event of the module, put or
// delete a key of its states `applied`, `schedule` and `findings`, and invoke one of three (`zfs.zfs_snapshots_record`,
// `zfs.zfs_snapshots_schedule` and `seat:node-service-manager.start`, of the two units of the module only). The root
// processes say everything they say on the bus through it; with the keeper not answering they act as with the bus down
// (A-8), and warn.
//
// Reading a key or the keys of those states (`get`, `keys`) is also served: the run reads the hand acts waiting in the
// state `schedule` and the apply reads the state `applied` (A-17), and a root process has no other way to the bus.

// KeeperSocketEnv names the variable holding the socket's path, in every process of the module that uses it.
const KeeperSocketEnv = "ZFS_KEEPER_SOCKET"

// KeeperTimeout bounds one call through the keeper.
const KeeperTimeout = 30 * time.Second

// The calls of the keeper.
const (
	OpEmit   = "emit"
	OpPut    = "put"
	OpDelete = "delete"
	OpGet    = "get"
	OpKeys   = "keys"
	OpInvoke = "invoke"
)

// KeeperRequest is one call: one JSON line.
type KeeperRequest struct {
	Op    string          `json:"op"`
	Kind  string          `json:"kind,omitempty"`
	State string          `json:"state,omitempty"`
	Key   string          `json:"key,omitempty"`
	Body  json.RawMessage `json:"body,omitempty"`
}

// KeeperAnswer is its answer: one JSON line.
type KeeperAnswer struct {
	Error string          `json:"error,omitempty"`
	Found bool            `json:"found,omitempty"`
	Value json.RawMessage `json:"value,omitempty"`
	Keys  []string        `json:"keys,omitempty"`
}

var keeperStates = map[string]bool{StateApplied: true, StateSchedule: true, StateFindings: true}

// Keeper serves the calls over the bus account of its own process.
type Keeper struct {
	Emit   func(kind string, body any) error
	States StateStore
	Call   func(key string, body any) (json.RawMessage, error)
	// PeerOf is the uid at the other end of a connection; nil reads it from the socket (SO_PEERCRED).
	PeerOf func(net.Conn) (int, error)
}

// invokable says whether a call key is one of the three, and for the service manager whether it starts one of the
// module's two units.
func invokable(key string, body json.RawMessage) error {
	switch {
	case key == ToolRecord || key == ToolSchedule:
		return nil
	case strings.HasPrefix(key, "seat:node-service-manager.start@"):
		var a map[string]any
		if json.Unmarshal(body, &a) != nil || a["scope"] != "system" || (a["unit"] != RunService && a["unit"] != ApplyUnit) || len(a) != 2 {
			return fmt.Errorf("the keeper starts only %s or %s, in the system scope", RunService, ApplyUnit)
		}
		return nil
	}
	return fmt.Errorf("%q is not one of the three calls the keeper invokes", key)
}

// Do answers one request.
func (k *Keeper) Do(r KeeperRequest) KeeperAnswer {
	fail := func(err error) KeeperAnswer { return KeeperAnswer{Error: err.Error()} }
	switch r.Op {
	case OpEmit:
		known := false
		for _, e := range EventKinds {
			known = known || (e == r.Kind && e != EventScrubRequested)
		}
		if !known {
			return fail(fmt.Errorf("%q is not an event the module's processes say", r.Kind))
		}
		var body any
		if len(r.Body) > 0 {
			if err := json.Unmarshal(r.Body, &body); err != nil {
				return fail(err)
			}
		}
		if err := k.Emit(r.Kind, body); err != nil {
			return fail(err)
		}
		return KeeperAnswer{}
	case OpPut, OpDelete, OpGet, OpKeys:
		if !keeperStates[r.State] {
			return fail(fmt.Errorf("%q is not a state of the module's processes (applied, schedule, findings)", r.State))
		}
		switch r.Op {
		case OpPut:
			var v any
			if err := json.Unmarshal(r.Body, &v); err != nil {
				return fail(err)
			}
			if err := k.States.Put(r.State, r.Key, v); err != nil {
				return fail(err)
			}
		case OpDelete:
			if err := k.States.Delete(r.State, r.Key); err != nil {
				return fail(err)
			}
		case OpGet:
			v, ok, err := k.States.Get(r.State, r.Key)
			if err != nil {
				return fail(err)
			}
			return KeeperAnswer{Found: ok, Value: v}
		case OpKeys:
			keys, err := k.States.Keys(r.State)
			if err != nil {
				return fail(err)
			}
			return KeeperAnswer{Keys: keys}
		}
		return KeeperAnswer{}
	case OpInvoke:
		if err := invokable(r.Key, r.Body); err != nil {
			return fail(err)
		}
		var body any
		_ = json.Unmarshal(r.Body, &body)
		out, err := k.Call(r.Key, body)
		if err != nil {
			return fail(err)
		}
		return KeeperAnswer{Value: out}
	}
	return fail(fmt.Errorf("%q is not a call of the keeper (emit, put, delete, get, keys, invoke)", r.Op))
}

// peerUID is the account at the other end of a unix socket.
func peerUID(c net.Conn) (int, error) {
	uc, ok := c.(*net.UnixConn)
	if !ok {
		return -1, errors.New("not a unix socket")
	}
	raw, err := uc.SyscallConn()
	if err != nil {
		return -1, err
	}
	uid := -1
	var gerr error
	if err := raw.Control(func(fd uintptr) {
		cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
		if err != nil {
			gerr = err
			return
		}
		uid = int(cred.Uid)
	}); err != nil {
		return -1, err
	}
	return uid, gerr
}

// Serve answers each connection on l, one request each, until ctx ends: a peer that is not root is told so and
// nothing it sent is read.
func (k *Keeper) Serve(ctx context.Context, l net.Listener) error {
	go func() { <-ctx.Done(); l.Close() }()
	peerOf := k.PeerOf
	if peerOf == nil {
		peerOf = peerUID
	}
	for {
		c, err := l.Accept()
		if err != nil {
			if ctx.Err() != nil {
				return nil
			}
			return err
		}
		go func() {
			defer c.Close()
			_ = c.SetDeadline(time.Now().Add(KeeperTimeout))
			enc := json.NewEncoder(c)
			if uid, err := peerOf(c); err != nil || uid != 0 {
				_ = enc.Encode(KeeperAnswer{Error: "only root opens the keeper's socket"})
				return
			}
			line, err := bufio.NewReader(c).ReadBytes('\n')
			if err != nil {
				return
			}
			var r KeeperRequest
			if err := json.Unmarshal(line, &r); err != nil {
				_ = enc.Encode(KeeperAnswer{Error: "the request is not JSON: " + err.Error()})
				return
			}
			_ = enc.Encode(k.Do(r))
		}()
	}
}

// Listen opens the socket: in a folder of the keeper's own, as the keeper's account, mode 0600 so that only the
// account itself and root open it; any older socket is replaced.
func Listen(sock string) (net.Listener, error) {
	if err := os.MkdirAll(filepath.Dir(sock), 0o711); err != nil {
		return nil, err
	}
	_ = os.Remove(sock)
	l, err := net.Listen("unix", sock)
	if err != nil {
		return nil, err
	}
	if err := os.Chmod(sock, 0o600); err != nil {
		l.Close()
		return nil, err
	}
	return l, nil
}

// keeperClient is a root process's way to the bus: calls on the keeper's socket.
type keeperClient struct {
	sock string
	// down, when set, is told of a call the keeper did not answer (not of an answer that is an error).
	down func(error)
}

func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) {
	a, err, unanswered := c.exchange(r)
	if unanswered && c.down != nil {
		c.down(err)
	}
	return a, err
}

// exchange is one call on the socket; unanswered says the keeper gave no answer (as opposed to an answer that is an error).
func (c keeperClient) exchange(r KeeperRequest) (a KeeperAnswer, err error, unanswered bool) {
	conn, err := net.DialTimeout("unix", c.sock, 5*time.Second)
	if err != nil {
		return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err), true
	}
	defer conn.Close()
	_ = conn.SetDeadline(time.Now().Add(KeeperTimeout))
	if err := json.NewEncoder(conn).Encode(r); err != nil {
		return KeeperAnswer{}, err, true
	}
	if err := json.NewDecoder(conn).Decode(&a); err != nil {
		return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err), true
	}
	if a.Error != "" {
		return a, errors.New(a.Error), false
	}
	return a, nil, false
}

func (c keeperClient) emit(kind string, body any) error {
	raw, err := json.Marshal(body)
	if err != nil {
		return err
	}
	_, err = c.do(KeeperRequest{Op: OpEmit, Kind: kind, Body: raw})
	return err
}

func (c keeperClient) Put(state, key string, v any) error {
	raw, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = c.do(KeeperRequest{Op: OpPut, State: state, Key: key, Body: raw})
	return err
}

func (c keeperClient) Delete(state, key string) error {
	_, err := c.do(KeeperRequest{Op: OpDelete, State: state, Key: key})
	return err
}

func (c keeperClient) Keys(state string) ([]string, error) {
	a, err := c.do(KeeperRequest{Op: OpKeys, State: state})
	return a.Keys, err
}

func (c keeperClient) Get(state, key string) (json.RawMessage, bool, error) {
	a, err := c.do(KeeperRequest{Op: OpGet, State: state, Key: key})
	return a.Value, a.Found, err
}

func (c keeperClient) call(key string, body any) (json.RawMessage, error) {
	raw, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	a, err := c.do(KeeperRequest{Op: OpInvoke, Key: key, Body: raw})
	return a.Value, err
}

// findingState is the state `findings` as the finding writer reaches it, through the keeper.
type findingState struct{ c keeperClient }

func (f findingState) Put(key string, value any) (uint64, error) {
	return 0, f.c.Put(StateFindings, key, value)
}
func (f findingState) Delete(key string) error { return f.c.Delete(StateFindings, key) }