313 lines
9.2 KiB
Go
313 lines
9.2 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// The keeper (design 42 zfs-owns-zfs A-37, §18, §20, §26, §27; ADR 0436 §1, §2).
|
|
//
|
|
// A process's bus account is a user the module makes, never root, so the three root processes (the apply, the run and
|
|
// the event follower) declare no `bus`. The process `zfs-keeper`, running as the module's user `zfs-keeper`, holds the
|
|
// module's one process bus account and serves a unix socket in the module's state directory that only root may open
|
|
// (the peer's uid is checked on every connection). Its calls are a closed list: emit an event of the module, put or
|
|
// delete a key of its states `applied`, `schedule` and `findings`, and invoke one of three (`zfs.zfs_snapshots_record`,
|
|
// `zfs.zfs_snapshots_schedule` and `seat:node-service-manager.start`, of the two units of the module only). The root
|
|
// processes say everything they say on the bus through it; with the keeper not answering they act as with the bus down
|
|
// (A-8), and warn.
|
|
//
|
|
// Reading a key or the keys of those states (`get`, `keys`) is also served: the run reads the hand acts waiting in the
|
|
// state `schedule` and the apply reads the state `applied` (A-17), and a root process has no other way to the bus.
|
|
|
|
// KeeperSocketEnv names the variable holding the socket's path, in every process of the module that uses it.
|
|
const KeeperSocketEnv = "ZFS_KEEPER_SOCKET"
|
|
|
|
// KeeperTimeout bounds one call through the keeper.
|
|
const KeeperTimeout = 30 * time.Second
|
|
|
|
// The calls of the keeper.
|
|
const (
|
|
OpEmit = "emit"
|
|
OpPut = "put"
|
|
OpDelete = "delete"
|
|
OpGet = "get"
|
|
OpKeys = "keys"
|
|
OpInvoke = "invoke"
|
|
)
|
|
|
|
// KeeperRequest is one call: one JSON line.
|
|
type KeeperRequest struct {
|
|
Op string `json:"op"`
|
|
Kind string `json:"kind,omitempty"`
|
|
State string `json:"state,omitempty"`
|
|
Key string `json:"key,omitempty"`
|
|
Body json.RawMessage `json:"body,omitempty"`
|
|
}
|
|
|
|
// KeeperAnswer is its answer: one JSON line.
|
|
type KeeperAnswer struct {
|
|
Error string `json:"error,omitempty"`
|
|
Found bool `json:"found,omitempty"`
|
|
Value json.RawMessage `json:"value,omitempty"`
|
|
Keys []string `json:"keys,omitempty"`
|
|
}
|
|
|
|
var keeperStates = map[string]bool{StateApplied: true, StateSchedule: true, StateFindings: true}
|
|
|
|
// Keeper serves the calls over the bus account of its own process.
|
|
type Keeper struct {
|
|
Emit func(kind string, body any) error
|
|
States StateStore
|
|
Call func(key string, body any) (json.RawMessage, error)
|
|
// PeerOf is the uid at the other end of a connection; nil reads it from the socket (SO_PEERCRED).
|
|
PeerOf func(net.Conn) (int, error)
|
|
}
|
|
|
|
// invokable says whether a call key is one of the three, and for the service manager whether it starts one of the
|
|
// module's two units.
|
|
func invokable(key string, body json.RawMessage) error {
|
|
switch {
|
|
case key == ToolRecord || key == ToolSchedule:
|
|
return nil
|
|
case strings.HasPrefix(key, "seat:node-service-manager.start@"):
|
|
var a map[string]any
|
|
if json.Unmarshal(body, &a) != nil || a["scope"] != "system" || (a["unit"] != RunService && a["unit"] != ApplyUnit) || len(a) != 2 {
|
|
return fmt.Errorf("the keeper starts only %s or %s, in the system scope", RunService, ApplyUnit)
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%q is not one of the three calls the keeper invokes", key)
|
|
}
|
|
|
|
// Do answers one request.
|
|
func (k *Keeper) Do(r KeeperRequest) KeeperAnswer {
|
|
fail := func(err error) KeeperAnswer { return KeeperAnswer{Error: err.Error()} }
|
|
switch r.Op {
|
|
case OpEmit:
|
|
known := false
|
|
for _, e := range EventKinds {
|
|
known = known || (e == r.Kind && e != EventScrubRequested)
|
|
}
|
|
if !known {
|
|
return fail(fmt.Errorf("%q is not an event the module's processes say", r.Kind))
|
|
}
|
|
var body any
|
|
if len(r.Body) > 0 {
|
|
if err := json.Unmarshal(r.Body, &body); err != nil {
|
|
return fail(err)
|
|
}
|
|
}
|
|
if err := k.Emit(r.Kind, body); err != nil {
|
|
return fail(err)
|
|
}
|
|
return KeeperAnswer{}
|
|
case OpPut, OpDelete, OpGet, OpKeys:
|
|
if !keeperStates[r.State] {
|
|
return fail(fmt.Errorf("%q is not a state of the module's processes (applied, schedule, findings)", r.State))
|
|
}
|
|
switch r.Op {
|
|
case OpPut:
|
|
var v any
|
|
if err := json.Unmarshal(r.Body, &v); err != nil {
|
|
return fail(err)
|
|
}
|
|
if err := k.States.Put(r.State, r.Key, v); err != nil {
|
|
return fail(err)
|
|
}
|
|
case OpDelete:
|
|
if err := k.States.Delete(r.State, r.Key); err != nil {
|
|
return fail(err)
|
|
}
|
|
case OpGet:
|
|
v, ok, err := k.States.Get(r.State, r.Key)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
return KeeperAnswer{Found: ok, Value: v}
|
|
case OpKeys:
|
|
keys, err := k.States.Keys(r.State)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
return KeeperAnswer{Keys: keys}
|
|
}
|
|
return KeeperAnswer{}
|
|
case OpInvoke:
|
|
if err := invokable(r.Key, r.Body); err != nil {
|
|
return fail(err)
|
|
}
|
|
var body any
|
|
_ = json.Unmarshal(r.Body, &body)
|
|
out, err := k.Call(r.Key, body)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
return KeeperAnswer{Value: out}
|
|
}
|
|
return fail(fmt.Errorf("%q is not a call of the keeper (emit, put, delete, get, keys, invoke)", r.Op))
|
|
}
|
|
|
|
// peerUID is the account at the other end of a unix socket.
|
|
func peerUID(c net.Conn) (int, error) {
|
|
uc, ok := c.(*net.UnixConn)
|
|
if !ok {
|
|
return -1, errors.New("not a unix socket")
|
|
}
|
|
raw, err := uc.SyscallConn()
|
|
if err != nil {
|
|
return -1, err
|
|
}
|
|
uid := -1
|
|
var gerr error
|
|
if err := raw.Control(func(fd uintptr) {
|
|
cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
|
|
if err != nil {
|
|
gerr = err
|
|
return
|
|
}
|
|
uid = int(cred.Uid)
|
|
}); err != nil {
|
|
return -1, err
|
|
}
|
|
return uid, gerr
|
|
}
|
|
|
|
// Serve answers each connection on l, one request each, until ctx ends: a peer that is not root is told so and
|
|
// nothing it sent is read.
|
|
func (k *Keeper) Serve(ctx context.Context, l net.Listener) error {
|
|
go func() { <-ctx.Done(); l.Close() }()
|
|
peerOf := k.PeerOf
|
|
if peerOf == nil {
|
|
peerOf = peerUID
|
|
}
|
|
for {
|
|
c, err := l.Accept()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
go func() {
|
|
defer c.Close()
|
|
_ = c.SetDeadline(time.Now().Add(KeeperTimeout))
|
|
enc := json.NewEncoder(c)
|
|
if uid, err := peerOf(c); err != nil || uid != 0 {
|
|
_ = enc.Encode(KeeperAnswer{Error: "only root opens the keeper's socket"})
|
|
return
|
|
}
|
|
line, err := bufio.NewReader(c).ReadBytes('\n')
|
|
if err != nil {
|
|
return
|
|
}
|
|
var r KeeperRequest
|
|
if err := json.Unmarshal(line, &r); err != nil {
|
|
_ = enc.Encode(KeeperAnswer{Error: "the request is not JSON: " + err.Error()})
|
|
return
|
|
}
|
|
_ = enc.Encode(k.Do(r))
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Listen opens the socket: in a folder of the keeper's own, as the keeper's account, mode 0600 so that only the
|
|
// account itself and root open it; any older socket is replaced.
|
|
func Listen(sock string) (net.Listener, error) {
|
|
if err := os.MkdirAll(filepath.Dir(sock), 0o711); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = os.Remove(sock)
|
|
l, err := net.Listen("unix", sock)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.Chmod(sock, 0o600); err != nil {
|
|
l.Close()
|
|
return nil, err
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
// keeperClient is a root process's way to the bus: calls on the keeper's socket.
|
|
type keeperClient struct{ sock string }
|
|
|
|
func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) {
|
|
conn, err := net.DialTimeout("unix", c.sock, 5*time.Second)
|
|
if err != nil {
|
|
return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err)
|
|
}
|
|
defer conn.Close()
|
|
_ = conn.SetDeadline(time.Now().Add(KeeperTimeout))
|
|
if err := json.NewEncoder(conn).Encode(r); err != nil {
|
|
return KeeperAnswer{}, err
|
|
}
|
|
var a KeeperAnswer
|
|
if err := json.NewDecoder(conn).Decode(&a); err != nil {
|
|
return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err)
|
|
}
|
|
if a.Error != "" {
|
|
return a, errors.New(a.Error)
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
func (c keeperClient) emit(kind string, body any) error {
|
|
raw, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = c.do(KeeperRequest{Op: OpEmit, Kind: kind, Body: raw})
|
|
return err
|
|
}
|
|
|
|
func (c keeperClient) Put(state, key string, v any) error {
|
|
raw, err := json.Marshal(v)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = c.do(KeeperRequest{Op: OpPut, State: state, Key: key, Body: raw})
|
|
return err
|
|
}
|
|
|
|
func (c keeperClient) Delete(state, key string) error {
|
|
_, err := c.do(KeeperRequest{Op: OpDelete, State: state, Key: key})
|
|
return err
|
|
}
|
|
|
|
func (c keeperClient) Keys(state string) ([]string, error) {
|
|
a, err := c.do(KeeperRequest{Op: OpKeys, State: state})
|
|
return a.Keys, err
|
|
}
|
|
|
|
func (c keeperClient) Get(state, key string) (json.RawMessage, bool, error) {
|
|
a, err := c.do(KeeperRequest{Op: OpGet, State: state, Key: key})
|
|
return a.Value, a.Found, err
|
|
}
|
|
|
|
func (c keeperClient) call(key string, body any) (json.RawMessage, error) {
|
|
raw, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
a, err := c.do(KeeperRequest{Op: OpInvoke, Key: key, Body: raw})
|
|
return a.Value, err
|
|
}
|
|
|
|
// findingState is the state `findings` as the finding writer reaches it, through the keeper.
|
|
type findingState struct{ c keeperClient }
|
|
|
|
func (f findingState) Put(key string, value any) (uint64, error) {
|
|
return 0, f.c.Put(StateFindings, key, value)
|
|
}
|
|
func (f findingState) Delete(key string) error { return f.c.Delete(StateFindings, key) }
|