mesh-catalog/modules/zfs/cmd/zfs-tools/keeper.go

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"golang.org/x/sys/unix"
)

// The keeper (design 42 zfs-owns-zfs A-37, §18, §20, §26, §27; ADR 0436 §1, §2).
//
// A process's bus account is a user the module makes, never root, so the three root processes (the apply, the run and
// the event follower) declare no `bus`. The process `zfs-keeper`, running as the module's user `zfs-keeper`, holds the
// module's one process bus account and serves a unix socket in the module's state directory that only root may open
// (the peer's uid is checked on every connection). Its calls are a closed list: emit an event of the module, put or
// delete a key of its states `applied`, `schedule` and `findings`, and invoke one of three (`zfs.zfs_snapshots_record`,
// `zfs.zfs_snapshots_schedule` and `seat:node-service-manager.start`, of the two units of the module only). The root
// processes say everything they say on the bus through it; with the keeper not answering they act as with the bus down
// (A-8), and warn.
//
// Reading a key or the keys of those states (`get`, `keys`) is also served: the run reads the hand acts waiting in the
// state `schedule` and the apply reads the state `applied` (A-17), and a root process has no other way to the bus.
//
// And exactly what G's findings writer needs, for the follower that runs it (design 47 G A-339, A-340): the publish of
// its three events on the module's finding stream (`finding`), and the hear of mesh-conditions' `findings-asked`
// (`hear-asks`), whose connection stays open, each ask one answer line after the first, empty one.

// KeeperSocketEnv names the variable holding the socket's path, in every process of the module that uses it.
const KeeperSocketEnv = "ZFS_KEEPER_SOCKET"

// KeeperTimeout bounds one call through the keeper.
const KeeperTimeout = 30 * time.Second

// The calls of the keeper.
const (
	OpEmit   = "emit"
	OpPut    = "put"
	OpDelete = "delete"
	OpGet    = "get"
	OpKeys   = "keys"
	OpInvoke = "invoke"
	// OpFinding says one event of the findings writer on the module's finding stream (A-339).
	OpFinding = "finding"
	// OpHearAsks hears mesh-conditions' findings-asked, one answer line per ask, until the connection closes (A-339, A-340).
	OpHearAsks = "hear-asks"
)

// FindingEvents are the events of G's findings writer, the ones the keeper says on the finding stream (A-340).
var FindingEvents = []string{"finding-written", "finding-cleared", "findings-stated"}

// KeeperRequest is one call: one JSON line.
type KeeperRequest struct {
	Op    string          `json:"op"`
	Kind  string          `json:"kind,omitempty"`
	State string          `json:"state,omitempty"`
	Key   string          `json:"key,omitempty"`
	Body  json.RawMessage `json:"body,omitempty"`
	// ID is a finding event's message id (OpFinding).
	ID string `json:"id,omitempty"`
}

// KeeperAnswer is its answer: one JSON line.
type KeeperAnswer struct {
	Error string          `json:"error,omitempty"`
	Found bool            `json:"found,omitempty"`
	Value json.RawMessage `json:"value,omitempty"`
	Keys  []string        `json:"keys,omitempty"`
}

var keeperStates = map[string]bool{StateApplied: true, StateSchedule: true, StateFindings: true}

// Keeper serves the calls over the bus account of its own process.
type Keeper struct {
	Emit   func(kind string, body any) error
	States StateStore
	Call   func(key string, body any) (json.RawMessage, error)
	// PeerOf is the uid at the other end of a connection; nil reads it from the socket (SO_PEERCRED).
	PeerOf func(net.Conn) (int, error)
	// Finding publishes one finding event on the module's finding stream, acknowledged, the id its message id (A-339).
	Finding func(event string, body json.RawMessage, id string) error
	// HearAsks hears mesh-conditions' findings-asked, each body handed to handle, until stop (A-339, A-340).
	HearAsks func(ctx context.Context, handle func([]byte)) (stop func(), err error)
}

// invokable says whether a call key is one of the three, and for the service manager whether it starts one of the
// module's two units.
func invokable(key string, body json.RawMessage) error {
	switch {
	case key == ToolRecord || key == ToolSchedule:
		return nil
	case strings.HasPrefix(key, "seat:node-service-manager.start@"):
		var a map[string]any
		if json.Unmarshal(body, &a) != nil || a["scope"] != "system" || (a["unit"] != RunService && a["unit"] != ApplyUnit) || len(a) != 2 {
			return fmt.Errorf("the keeper starts only %s or %s, in the system scope", RunService, ApplyUnit)
		}
		return nil
	}
	return fmt.Errorf("%q is not one of the three calls the keeper invokes", key)
}

// Do answers one request.
func (k *Keeper) Do(r KeeperRequest) KeeperAnswer {
	fail := func(err error) KeeperAnswer { return KeeperAnswer{Error: err.Error()} }
	switch r.Op {
	case OpEmit:
		known := false
		for _, e := range EventKinds {
			known = known || (e == r.Kind && e != EventScrubRequested)
		}
		if !known {
			return fail(fmt.Errorf("%q is not an event the module's processes say", r.Kind))
		}
		var body any
		if len(r.Body) > 0 {
			if err := json.Unmarshal(r.Body, &body); err != nil {
				return fail(err)
			}
		}
		if err := k.Emit(r.Kind, body); err != nil {
			return fail(err)
		}
		return KeeperAnswer{}
	case OpPut, OpDelete, OpGet, OpKeys:
		if !keeperStates[r.State] {
			return fail(fmt.Errorf("%q is not a state of the module's processes (applied, schedule, findings)", r.State))
		}
		switch r.Op {
		case OpPut:
			var v any
			if err := json.Unmarshal(r.Body, &v); err != nil {
				return fail(err)
			}
			if err := k.States.Put(r.State, r.Key, v); err != nil {
				return fail(err)
			}
		case OpDelete:
			if err := k.States.Delete(r.State, r.Key); err != nil {
				return fail(err)
			}
		case OpGet:
			v, ok, err := k.States.Get(r.State, r.Key)
			if err != nil {
				return fail(err)
			}
			return KeeperAnswer{Found: ok, Value: v}
		case OpKeys:
			keys, err := k.States.Keys(r.State)
			if err != nil {
				return fail(err)
			}
			return KeeperAnswer{Keys: keys}
		}
		return KeeperAnswer{}
	case OpInvoke:
		if err := invokable(r.Key, r.Body); err != nil {
			return fail(err)
		}
		var body any
		_ = json.Unmarshal(r.Body, &body)
		out, err := k.Call(r.Key, body)
		if err != nil {
			return fail(err)
		}
		return KeeperAnswer{Value: out}
	case OpFinding:
		if !slices.Contains(FindingEvents, r.Kind) {
			return fail(fmt.Errorf("%q is not an event of the findings writer (%s)", r.Kind, strings.Join(FindingEvents, ", ")))
		}
		if k.Finding == nil {
			return fail(errors.New("this keeper says no finding"))
		}
		if err := k.Finding(r.Kind, r.Body, r.ID); err != nil {
			return fail(err)
		}
		return KeeperAnswer{}
	}
	return fail(fmt.Errorf("%q is not a call of the keeper (emit, put, delete, get, keys, invoke, finding, hear-asks)", r.Op))
}

// peerUID is the account at the other end of a unix socket.
func peerUID(c net.Conn) (int, error) {
	uc, ok := c.(*net.UnixConn)
	if !ok {
		return -1, errors.New("not a unix socket")
	}
	raw, err := uc.SyscallConn()
	if err != nil {
		return -1, err
	}
	uid := -1
	var gerr error
	if err := raw.Control(func(fd uintptr) {
		cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
		if err != nil {
			gerr = err
			return
		}
		uid = int(cred.Uid)
	}); err != nil {
		return -1, err
	}
	return uid, gerr
}

// Serve answers each connection on l, one request each, until ctx ends: a peer that is not root is told so and
// nothing it sent is read.
func (k *Keeper) Serve(ctx context.Context, l net.Listener) error {
	go func() { <-ctx.Done(); l.Close() }()
	peerOf := k.PeerOf
	if peerOf == nil {
		peerOf = peerUID
	}
	for {
		c, err := l.Accept()
		if err != nil {
			if ctx.Err() != nil {
				return nil
			}
			return err
		}
		go func() {
			defer c.Close()
			_ = c.SetDeadline(time.Now().Add(KeeperTimeout))
			enc := json.NewEncoder(c)
			if uid, err := peerOf(c); err != nil || uid != 0 {
				_ = enc.Encode(KeeperAnswer{Error: "only root opens the keeper's socket"})
				return
			}
			line, err := bufio.NewReader(c).ReadBytes('\n')
			if err != nil {
				return
			}
			var r KeeperRequest
			if err := json.Unmarshal(line, &r); err != nil {
				_ = enc.Encode(KeeperAnswer{Error: "the request is not JSON: " + err.Error()})
				return
			}
			if r.Op == OpHearAsks {
				k.hear(ctx, c, enc)
				return
			}
			_ = enc.Encode(k.Do(r))
		}()
	}
}

// hear serves one hear of findings-asked: a first, empty answer once the hear is made (or its error), then one answer per
// ask, until the peer closes the connection or ctx ends. The connection has no deadline while it hears.
func (k *Keeper) hear(ctx context.Context, c net.Conn, enc *json.Encoder) {
	if k.HearAsks == nil {
		_ = enc.Encode(KeeperAnswer{Error: "this keeper hears no asks"})
		return
	}
	_ = c.SetDeadline(time.Time{})
	hctx, cancel := context.WithCancel(ctx)
	defer cancel()
	var mu sync.Mutex
	mu.Lock() // no ask is written before the first answer
	stop, err := k.HearAsks(hctx, func(body []byte) {
		mu.Lock()
		defer mu.Unlock()
		_ = enc.Encode(KeeperAnswer{Value: body})
	})
	if err != nil {
		_ = enc.Encode(KeeperAnswer{Error: err.Error()})
		mu.Unlock()
		return
	}
	defer stop()
	_ = enc.Encode(KeeperAnswer{})
	mu.Unlock()
	go func() { <-hctx.Done(); c.Close() }()
	_, _ = io.Copy(io.Discard, c) // until the peer closes
}

// Listen opens the socket: in a folder of the keeper's own, as the keeper's account, mode 0600 so that only the
// account itself and root open it; any older socket is replaced.
func Listen(sock string) (net.Listener, error) {
	if err := os.MkdirAll(filepath.Dir(sock), 0o711); err != nil {
		return nil, err
	}
	_ = os.Remove(sock)
	l, err := net.Listen("unix", sock)
	if err != nil {
		return nil, err
	}
	if err := os.Chmod(sock, 0o600); err != nil {
		l.Close()
		return nil, err
	}
	return l, nil
}

// keeperClient is a root process's way to the bus: calls on the keeper's socket.
type keeperClient struct {
	sock string
	// down, when set, is told of a call the keeper did not answer (not of an answer that is an error).
	down func(error)
}

func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) {
	a, err, unanswered := c.exchange(r)
	if unanswered && c.down != nil {
		c.down(err)
	}
	return a, err
}

// exchange is one call on the socket; unanswered says the keeper gave no answer (as opposed to an answer that is an error).
func (c keeperClient) exchange(r KeeperRequest) (a KeeperAnswer, err error, unanswered bool) {
	conn, err := net.DialTimeout("unix", c.sock, 5*time.Second)
	if err != nil {
		return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err), true
	}
	defer conn.Close()
	_ = conn.SetDeadline(time.Now().Add(KeeperTimeout))
	if err := json.NewEncoder(conn).Encode(r); err != nil {
		return KeeperAnswer{}, err, true
	}
	if err := json.NewDecoder(conn).Decode(&a); err != nil {
		return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err), true
	}
	if a.Error != "" {
		return a, errors.New(a.Error), false
	}
	return a, nil, false
}

func (c keeperClient) emit(kind string, body any) error {
	raw, err := json.Marshal(body)
	if err != nil {
		return err
	}
	_, err = c.do(KeeperRequest{Op: OpEmit, Kind: kind, Body: raw})
	return err
}

func (c keeperClient) Put(state, key string, v any) error {
	raw, err := json.Marshal(v)
	if err != nil {
		return err
	}
	_, err = c.do(KeeperRequest{Op: OpPut, State: state, Key: key, Body: raw})
	return err
}

func (c keeperClient) Delete(state, key string) error {
	_, err := c.do(KeeperRequest{Op: OpDelete, State: state, Key: key})
	return err
}

func (c keeperClient) Keys(state string) ([]string, error) {
	a, err := c.do(KeeperRequest{Op: OpKeys, State: state})
	return a.Keys, err
}

func (c keeperClient) Get(state, key string) (json.RawMessage, bool, error) {
	a, err := c.do(KeeperRequest{Op: OpGet, State: state, Key: key})
	return a.Value, a.Found, err
}

func (c keeperClient) call(key string, body any) (json.RawMessage, error) {
	raw, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	a, err := c.do(KeeperRequest{Op: OpInvoke, Key: key, Body: raw})
	return a.Value, err
}

// finding says one finding event through the keeper (A-339).
func (c keeperClient) finding(event string, body []byte, id string) error {
	_, err := c.do(KeeperRequest{Op: OpFinding, Kind: event, Body: body, ID: id})
	return err
}

// hearAsks opens the keeper's hear of findings-asked and hands each ask to handle, until stop; ended is told when the
// keeper ends it first (A-339).
func (c keeperClient) hearAsks(handle func([]byte), ended func(error)) (stop func(), err error) {
	conn, err := net.DialTimeout("unix", c.sock, 5*time.Second)
	if err != nil {
		return nil, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err)
	}
	_ = conn.SetDeadline(time.Now().Add(KeeperTimeout))
	if err := json.NewEncoder(conn).Encode(KeeperRequest{Op: OpHearAsks}); err != nil {
		conn.Close()
		return nil, err
	}
	dec := json.NewDecoder(conn)
	var first KeeperAnswer
	if err := dec.Decode(&first); err != nil {
		conn.Close()
		return nil, fmt.Errorf("the keeper's answer: %v", err)
	}
	if first.Error != "" {
		conn.Close()
		return nil, errors.New(first.Error)
	}
	_ = conn.SetDeadline(time.Time{})
	var stopped atomic.Bool
	go func() {
		for {
			var a KeeperAnswer
			if err := dec.Decode(&a); err != nil {
				if !stopped.Load() && ended != nil {
					ended(err)
				}
				return
			}
			if len(a.Value) > 0 {
				handle(a.Value)
			}
		}
	}()
	return func() { stopped.Store(true); conn.Close() }, nil
}

// keeperFindings is the state `findings` as G's findings writer reaches it, through the keeper (views.FindingState).
type keeperFindings struct{ c keeperClient }

func (f keeperFindings) Get(key string) (*stdio.StateEntry, error) {
	raw, ok, err := f.c.Get(StateFindings, key)
	if err != nil || !ok {
		return nil, err
	}
	return &stdio.StateEntry{Key: key, Value: raw}, nil
}
func (f keeperFindings) Put(key string, value any) (uint64, error) {
	return 0, f.c.Put(StateFindings, key, value)
}
func (f keeperFindings) Delete(key string) error { return f.c.Delete(StateFindings, key) }
func (f keeperFindings) Keys() ([]string, error) { return f.c.Keys(StateFindings) }