package main import ( "bufio" "context" "encoding/json" "errors" "fmt" "io" "net" "os" "path/filepath" "slices" "strings" "sync" "sync/atomic" "time" stdio "git.novox.be/novox/mesh-sdk/go" "golang.org/x/sys/unix" ) // The keeper (design 42 zfs-owns-zfs A-37, §18, §20, §26, §27; ADR 0436 §1, §2). // // A process's bus account is a user the module makes, never root, so the three root processes (the apply, the run and // the event follower) declare no `bus`. The process `zfs-keeper`, running as the module's user `zfs-keeper`, holds the // module's one process bus account and serves a unix socket in the module's state directory that only root may open // (the peer's uid is checked on every connection). Its calls are a closed list: emit an event of the module, put or // delete a key of its states `applied`, `schedule` and `findings`, and invoke one of three (`zfs.zfs_snapshots_record`, // `zfs.zfs_snapshots_schedule` and `seat:node-service-manager.start`, of the two units of the module only). The root // processes say everything they say on the bus through it; with the keeper not answering they act as with the bus down // (A-8), and warn. // // Reading a key or the keys of those states (`get`, `keys`) is also served: the run reads the hand acts waiting in the // state `schedule` and the apply reads the state `applied` (A-17), and a root process has no other way to the bus. // // And exactly what G's findings writer needs, for the follower that runs it (design 47 G A-339, A-340): the publish of // its three events on the module's finding stream (`finding`), and the hear of mesh-conditions' `findings-asked` // (`hear-asks`), whose connection stays open, each ask one answer line after the first, empty one. // KeeperSocketEnv names the variable holding the socket's path, in every process of the module that uses it. const KeeperSocketEnv = "ZFS_KEEPER_SOCKET" // KeeperTimeout bounds one call through the keeper. const KeeperTimeout = 30 * time.Second // The calls of the keeper. const ( OpEmit = "emit" OpPut = "put" OpDelete = "delete" OpGet = "get" OpKeys = "keys" OpInvoke = "invoke" // OpFinding says one event of the findings writer on the module's finding stream (A-339). OpFinding = "finding" // OpHearAsks hears mesh-conditions' findings-asked, one answer line per ask, until the connection closes (A-339, A-340). OpHearAsks = "hear-asks" ) // FindingEvents are the events of G's findings writer, the ones the keeper says on the finding stream (A-340). var FindingEvents = []string{"finding-written", "finding-cleared", "findings-stated"} // KeeperRequest is one call: one JSON line. type KeeperRequest struct { Op string `json:"op"` Kind string `json:"kind,omitempty"` State string `json:"state,omitempty"` Key string `json:"key,omitempty"` Body json.RawMessage `json:"body,omitempty"` // ID is a finding event's message id (OpFinding). ID string `json:"id,omitempty"` } // KeeperAnswer is its answer: one JSON line. type KeeperAnswer struct { Error string `json:"error,omitempty"` Found bool `json:"found,omitempty"` Value json.RawMessage `json:"value,omitempty"` Keys []string `json:"keys,omitempty"` } var keeperStates = map[string]bool{StateApplied: true, StateSchedule: true, StateFindings: true} // Keeper serves the calls over the bus account of its own process. type Keeper struct { Emit func(kind string, body any) error States StateStore Call func(key string, body any) (json.RawMessage, error) // PeerOf is the uid at the other end of a connection; nil reads it from the socket (SO_PEERCRED). PeerOf func(net.Conn) (int, error) // Finding publishes one finding event on the module's finding stream, acknowledged, the id its message id (A-339). Finding func(event string, body json.RawMessage, id string) error // HearAsks hears mesh-conditions' findings-asked, each body handed to handle, until stop (A-339, A-340). HearAsks func(ctx context.Context, handle func([]byte)) (stop func(), err error) } // invokable says whether a call key is one of the three, and for the service manager whether it starts one of the // module's two units. func invokable(key string, body json.RawMessage) error { switch { case key == ToolRecord || key == ToolSchedule: return nil case strings.HasPrefix(key, "seat:node-service-manager.start@"): var a map[string]any if json.Unmarshal(body, &a) != nil || a["scope"] != "system" || (a["unit"] != RunService && a["unit"] != ApplyUnit) || len(a) != 2 { return fmt.Errorf("the keeper starts only %s or %s, in the system scope", RunService, ApplyUnit) } return nil } return fmt.Errorf("%q is not one of the three calls the keeper invokes", key) } // Do answers one request. func (k *Keeper) Do(r KeeperRequest) KeeperAnswer { fail := func(err error) KeeperAnswer { return KeeperAnswer{Error: err.Error()} } switch r.Op { case OpEmit: known := false for _, e := range EventKinds { known = known || (e == r.Kind && e != EventScrubRequested) } if !known { return fail(fmt.Errorf("%q is not an event the module's processes say", r.Kind)) } var body any if len(r.Body) > 0 { if err := json.Unmarshal(r.Body, &body); err != nil { return fail(err) } } if err := k.Emit(r.Kind, body); err != nil { return fail(err) } return KeeperAnswer{} case OpPut, OpDelete, OpGet, OpKeys: if !keeperStates[r.State] { return fail(fmt.Errorf("%q is not a state of the module's processes (applied, schedule, findings)", r.State)) } switch r.Op { case OpPut: var v any if err := json.Unmarshal(r.Body, &v); err != nil { return fail(err) } if err := k.States.Put(r.State, r.Key, v); err != nil { return fail(err) } case OpDelete: if err := k.States.Delete(r.State, r.Key); err != nil { return fail(err) } case OpGet: v, ok, err := k.States.Get(r.State, r.Key) if err != nil { return fail(err) } return KeeperAnswer{Found: ok, Value: v} case OpKeys: keys, err := k.States.Keys(r.State) if err != nil { return fail(err) } return KeeperAnswer{Keys: keys} } return KeeperAnswer{} case OpInvoke: if err := invokable(r.Key, r.Body); err != nil { return fail(err) } var body any _ = json.Unmarshal(r.Body, &body) out, err := k.Call(r.Key, body) if err != nil { return fail(err) } return KeeperAnswer{Value: out} case OpFinding: if !slices.Contains(FindingEvents, r.Kind) { return fail(fmt.Errorf("%q is not an event of the findings writer (%s)", r.Kind, strings.Join(FindingEvents, ", "))) } if k.Finding == nil { return fail(errors.New("this keeper says no finding")) } if err := k.Finding(r.Kind, r.Body, r.ID); err != nil { return fail(err) } return KeeperAnswer{} } return fail(fmt.Errorf("%q is not a call of the keeper (emit, put, delete, get, keys, invoke, finding, hear-asks)", r.Op)) } // peerUID is the account at the other end of a unix socket. func peerUID(c net.Conn) (int, error) { uc, ok := c.(*net.UnixConn) if !ok { return -1, errors.New("not a unix socket") } raw, err := uc.SyscallConn() if err != nil { return -1, err } uid := -1 var gerr error if err := raw.Control(func(fd uintptr) { cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) if err != nil { gerr = err return } uid = int(cred.Uid) }); err != nil { return -1, err } return uid, gerr } // Serve answers each connection on l, one request each, until ctx ends: a peer that is not root is told so and // nothing it sent is read. func (k *Keeper) Serve(ctx context.Context, l net.Listener) error { go func() { <-ctx.Done(); l.Close() }() peerOf := k.PeerOf if peerOf == nil { peerOf = peerUID } for { c, err := l.Accept() if err != nil { if ctx.Err() != nil { return nil } return err } go func() { defer c.Close() _ = c.SetDeadline(time.Now().Add(KeeperTimeout)) enc := json.NewEncoder(c) if uid, err := peerOf(c); err != nil || uid != 0 { _ = enc.Encode(KeeperAnswer{Error: "only root opens the keeper's socket"}) return } line, err := bufio.NewReader(c).ReadBytes('\n') if err != nil { return } var r KeeperRequest if err := json.Unmarshal(line, &r); err != nil { _ = enc.Encode(KeeperAnswer{Error: "the request is not JSON: " + err.Error()}) return } if r.Op == OpHearAsks { k.hear(ctx, c, enc) return } _ = enc.Encode(k.Do(r)) }() } } // hear serves one hear of findings-asked: a first, empty answer once the hear is made (or its error), then one answer per // ask, until the peer closes the connection or ctx ends. The connection has no deadline while it hears. func (k *Keeper) hear(ctx context.Context, c net.Conn, enc *json.Encoder) { if k.HearAsks == nil { _ = enc.Encode(KeeperAnswer{Error: "this keeper hears no asks"}) return } _ = c.SetDeadline(time.Time{}) hctx, cancel := context.WithCancel(ctx) defer cancel() var mu sync.Mutex mu.Lock() // no ask is written before the first answer stop, err := k.HearAsks(hctx, func(body []byte) { mu.Lock() defer mu.Unlock() _ = enc.Encode(KeeperAnswer{Value: body}) }) if err != nil { _ = enc.Encode(KeeperAnswer{Error: err.Error()}) mu.Unlock() return } defer stop() _ = enc.Encode(KeeperAnswer{}) mu.Unlock() go func() { <-hctx.Done(); c.Close() }() _, _ = io.Copy(io.Discard, c) // until the peer closes } // Listen opens the socket: in a folder of the keeper's own, as the keeper's account, mode 0600 so that only the // account itself and root open it; any older socket is replaced. func Listen(sock string) (net.Listener, error) { if err := os.MkdirAll(filepath.Dir(sock), 0o711); err != nil { return nil, err } _ = os.Remove(sock) l, err := net.Listen("unix", sock) if err != nil { return nil, err } if err := os.Chmod(sock, 0o600); err != nil { l.Close() return nil, err } return l, nil } // keeperClient is a root process's way to the bus: calls on the keeper's socket. type keeperClient struct { sock string // down, when set, is told of a call the keeper did not answer (not of an answer that is an error). down func(error) } func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) { a, err, unanswered := c.exchange(r) if unanswered && c.down != nil { c.down(err) } return a, err } // exchange is one call on the socket; unanswered says the keeper gave no answer (as opposed to an answer that is an error). func (c keeperClient) exchange(r KeeperRequest) (a KeeperAnswer, err error, unanswered bool) { conn, err := net.DialTimeout("unix", c.sock, 5*time.Second) if err != nil { return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err), true } defer conn.Close() _ = conn.SetDeadline(time.Now().Add(KeeperTimeout)) if err := json.NewEncoder(conn).Encode(r); err != nil { return KeeperAnswer{}, err, true } if err := json.NewDecoder(conn).Decode(&a); err != nil { return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err), true } if a.Error != "" { return a, errors.New(a.Error), false } return a, nil, false } func (c keeperClient) emit(kind string, body any) error { raw, err := json.Marshal(body) if err != nil { return err } _, err = c.do(KeeperRequest{Op: OpEmit, Kind: kind, Body: raw}) return err } func (c keeperClient) Put(state, key string, v any) error { raw, err := json.Marshal(v) if err != nil { return err } _, err = c.do(KeeperRequest{Op: OpPut, State: state, Key: key, Body: raw}) return err } func (c keeperClient) Delete(state, key string) error { _, err := c.do(KeeperRequest{Op: OpDelete, State: state, Key: key}) return err } func (c keeperClient) Keys(state string) ([]string, error) { a, err := c.do(KeeperRequest{Op: OpKeys, State: state}) return a.Keys, err } func (c keeperClient) Get(state, key string) (json.RawMessage, bool, error) { a, err := c.do(KeeperRequest{Op: OpGet, State: state, Key: key}) return a.Value, a.Found, err } func (c keeperClient) call(key string, body any) (json.RawMessage, error) { raw, err := json.Marshal(body) if err != nil { return nil, err } a, err := c.do(KeeperRequest{Op: OpInvoke, Key: key, Body: raw}) return a.Value, err } // finding says one finding event through the keeper (A-339). func (c keeperClient) finding(event string, body []byte, id string) error { _, err := c.do(KeeperRequest{Op: OpFinding, Kind: event, Body: body, ID: id}) return err } // hearAsks opens the keeper's hear of findings-asked and hands each ask to handle, until stop; ended is told when the // keeper ends it first (A-339). func (c keeperClient) hearAsks(handle func([]byte), ended func(error)) (stop func(), err error) { conn, err := net.DialTimeout("unix", c.sock, 5*time.Second) if err != nil { return nil, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err) } _ = conn.SetDeadline(time.Now().Add(KeeperTimeout)) if err := json.NewEncoder(conn).Encode(KeeperRequest{Op: OpHearAsks}); err != nil { conn.Close() return nil, err } dec := json.NewDecoder(conn) var first KeeperAnswer if err := dec.Decode(&first); err != nil { conn.Close() return nil, fmt.Errorf("the keeper's answer: %v", err) } if first.Error != "" { conn.Close() return nil, errors.New(first.Error) } _ = conn.SetDeadline(time.Time{}) var stopped atomic.Bool go func() { for { var a KeeperAnswer if err := dec.Decode(&a); err != nil { if !stopped.Load() && ended != nil { ended(err) } return } if len(a.Value) > 0 { handle(a.Value) } } }() return func() { stopped.Store(true); conn.Close() }, nil } // keeperFindings is the state `findings` as G's findings writer reaches it, through the keeper (views.FindingState). type keeperFindings struct{ c keeperClient } func (f keeperFindings) Get(key string) (*stdio.StateEntry, error) { raw, ok, err := f.c.Get(StateFindings, key) if err != nil || !ok { return nil, err } return &stdio.StateEntry{Key: key, Value: raw}, nil } func (f keeperFindings) Put(key string, value any) (uint64, error) { return 0, f.c.Put(StateFindings, key, value) } func (f keeperFindings) Delete(key string) error { return f.c.Delete(StateFindings, key) } func (f keeperFindings) Keys() ([]string, error) { return f.c.Keys(StateFindings) }