mesh-catalog/modules/nfs-server/cmd/nfs-server/tools.go

package main

import (
	"context"
	"fmt"
	"net/netip"
	"path"
	"sort"
	"strings"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

// Seat is the role this module holds on its machine (novox/hq ADR 0263).
const Seat = "node-nfs-server"

// Stale is how old the process's last look may be before the tools say it stopped looking.
const Stale = 5 * time.Minute

// verb is one of the seat's verbs, listed as `<seat>.<verb>` so the runtime serves it on the seat's
// subject, as <node>/node-nfs-server.<verb>.
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
	return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
}

func str(description string) map[string]any {
	return map[string]any{"type": "string", "description": description}
}

func arg(a map[string]any, k string) string {
	v, _ := a[k].(string)
	return strings.TrimSpace(v)
}

func flag(a map[string]any, k string) bool {
	switch v := a[k].(type) {
	case bool:
		return v
	case string:
		return strings.EqualFold(strings.TrimSpace(v), "true")
	}
	return false
}

// confirmed says whether an act was confirmed: the boolean true, or the string "true" the seat's schema
// passes.
func confirmed(a map[string]any) bool { return flag(a, "confirm") }

// Tools are the seat's verbs and the module's health.
func Tools(m *Machine) []stdio.Tool {
	return []stdio.Tool{
		verb("exports", "Every share this machine exports for the mesh: its name, folder, rw or ro, the owner every "+
			"client is mapped to (uid, gid), the nodes it is exported to (each granted by this machine and wanting it, "+
			"at its private address, with its access) and whether the kernel holds it now; what does not match "+
			"between the grants and the wants; the shares the assignment asked for that were refused, and why; and every export "+
			"on the machine that is not the mesh's (/etc/exports, the other files in /etc/exports.d, a ZFS dataset's "+
			"sharenfs). Replaces exportfs -v, cat /etc/exports and zfs get sharenfs. (r)",
			map[string]any{}, func(map[string]any) (any, error) { return m.Exports(), nil }),
		verb("clients", "Which machines have a share of this machine mounted now: each NFS client the kernel knows, "+
			"with its address and the name it gave, or — when the kernel's list cannot be read — the connections "+
			"to port 2049. Replaces cat /proc/fs/nfsd/clients/*/info and ss -tn sport = :2049. (r)",
			map[string]any{}, func(map[string]any) (any, error) { return m.Clients(), nil }),
		verb("test", "Whether this machine exports one share for the mesh now: the share is asked for and not "+
			"refused, the kernel holds it for every node it is exported to, and the NFS server is active; with an "+
			"address, also whether that address is inside the range and the share is exported to it. Replaces showmount -e. (r)",
			map[string]any{"share": str("the share's name"), "address": str("a client's address on the private network (optional)")},
			func(a map[string]any) (any, error) { return m.Test(arg(a, "share"), arg(a, "address")) }),
		verb("reload", "Have the kernel read the export files again now (exportfs -ra), and answer the mesh's shares "+
			"as it then holds them. The module's process does this itself whenever its file changes or a share is "+
			"missing; this is for after a change made outside it. Replaces exportfs -ra. (a)",
			map[string]any{}, func(map[string]any) (any, error) { return m.Reload() }),
		verb("adopt", "Take over the export a ZFS dataset's sharenfs property makes for a folder the mesh now "+
			"exports: sets sharenfs=off on that one dataset and nothing else, then exportfs -ra. Only when the "+
			"mesh's own export of the same folder is live in the kernel to at least one node, so the folder is never "+
			"unexported for everyone; unless confirm is true, a dry run that says what it would do. A person's act, with why. Replaces zfs set "+
			"sharenfs=off. (a)",
			map[string]any{
				"path":    str("the shared folder, as the share names it"),
				"confirm": str("\"true\" to do it; anything else, a dry run"),
				"why":     str("why, in a sentence (required with confirm)"),
			},
			func(a map[string]any) (any, error) { return m.Adopt(arg(a, "path"), confirmed(a), arg(a, "why")) }),
		{
			Name: "nfs_health",
			Description: "Whether this machine's shares are served, as the module's health check (novox/hq ADR 0240, " +
				"ADR 0263): unhealthy when the NFS server is not active, the module's process has not looked for five " +
				"minutes, a share the assignment asks for is refused or missing from the kernel's table, or the " +
				"export file could not be written or read again; with why, one sentence per finding. (r)",
			Run: func(map[string]any) (any, error) { return m.Health(), nil },
		},
	}
}

// ExportsAnswer is what `exports` answers.
type ExportsAnswer struct {
	Shares  []Exported `json:"shares"`
	Refused []Refused  `json:"refused,omitempty"`
	Foreign []Entry    `json:"foreign"`
	Service string     `json:"service"`
	Range   string     `json:"range,omitempty"`
	// Unmatched is what does not match between this machine's grants and the nodes' wants.
	Unmatched []string `json:"unmatched,omitempty"`
	Looked    string   `json:"looked,omitempty"`
	Said      []string `json:"said"`
	Notes     []string `json:"notes,omitempty"`
}

// Exports is the machine's exports now.
func (m *Machine) Exports() ExportsAnswer {
	a := ExportsAnswer{Shares: []Exported{}, Foreign: []Entry{}, Service: m.ServiceState()}
	st, err := m.LastStatus()
	if err != nil {
		a.Notes = append(a.Notes, err.Error())
	} else {
		a.Range, a.Refused = st.Range, st.Refused
		a.Looked = st.At.Format(time.RFC3339)
		live, note := m.Live()
		if note != "" {
			a.Notes = append(a.Notes, note)
		}
		for _, e := range st.Shares {
			if note == "" {
				e.Live = SharedLive(live, e)
			}
			a.Shares = append(a.Shares, e)
		}
		a.Notes = append(a.Notes, st.Problems...)
		a.Unmatched = st.Said
	}
	entries, sets, notes := m.Foreign()
	a.Notes = append(a.Notes, notes...)
	a.Foreign = append(a.Foreign, entries...)
	for _, d := range sets {
		a.Foreign = append(a.Foreign, Entry{Path: d.Mountpoint, Client: d.ShareNFS, From: "zfs sharenfs on " + d.Name})
	}
	for _, e := range a.Shares {
		if len(e.To) == 0 {
			a.Said = append(a.Said, fmt.Sprintf("%s: %s, exported to no node", e.Name, e.Path))
			continue
		}
		state := "live"
		if !e.Live {
			state = "NOT in the kernel's table"
		}
		a.Said = append(a.Said, fmt.Sprintf("%s: %s to %s, every client as %d:%d — %s", e.Name, e.Path, grantees(e.To), e.UID, e.GID, state))
	}
	a.Said = append(a.Said, a.Unmatched...)
	for _, r := range a.Refused {
		a.Said = append(a.Said, "refused: "+r.Why)
	}
	for _, f := range a.Foreign {
		a.Said = append(a.Said, fmt.Sprintf("not the mesh's: %s to %s (%s)", f.Path, f.Client, f.From))
	}
	if len(a.Shares) == 0 && len(a.Refused) == 0 {
		a.Said = append(a.Said, "the mesh exports nothing here")
	}
	a.Said = append(a.Said, "the NFS server is "+a.Service)
	return a
}

// Client is one machine that has a share mounted.
type Client struct {
	Address string `json:"address"`
	Name    string `json:"name,omitempty"`
	Minor   string `json:"minor_version,omitempty"`
}

// ClientsAnswer is what `clients` answers.
type ClientsAnswer struct {
	Clients []Client `json:"clients"`
	From    string   `json:"from"`
	Notes   []string `json:"notes,omitempty"`
}

// ParseClientInfo reads one /proc/fs/nfsd/clients/<id>/info.
func ParseClientInfo(text string) Client {
	var c Client
	for _, line := range strings.Split(text, "\n") {
		k, v, ok := strings.Cut(line, ":")
		if !ok {
			continue
		}
		v = strings.Trim(strings.TrimSpace(v), `"`)
		switch strings.TrimSpace(k) {
		case "address":
			c.Address = v
		case "name":
			c.Name = v
		case "minor version":
			c.Minor = v
		}
	}
	return c
}

// Clients are the NFS clients the kernel knows, or the connections to 2049.
func (m *Machine) Clients() ClientsAnswer {
	a := ClientsAnswer{Clients: []Client{}}
	const dir = "/proc/fs/nfsd/clients"
	infos := m.Glob(dir + "/*/info")
	if len(infos) == 0 && m.UID != 0 {
		if r, err := m.AsRoot("find", dir, "-maxdepth", "2", "-name", "info"); err == nil {
			for _, l := range strings.Fields(r.Out) {
				infos = append(infos, l)
			}
		} else {
			a.Notes = append(a.Notes, err.Error())
		}
	}
	for _, f := range infos {
		text, err := m.Read(f)
		if err != nil && m.UID != 0 {
			r, e := m.AsRoot("cat", f)
			text, err = r.Out, e
		}
		if err != nil {
			a.Notes = append(a.Notes, f+": "+err.Error())
			continue
		}
		a.Clients = append(a.Clients, ParseClientInfo(text))
	}
	if len(infos) > 0 {
		a.From = dir
		return a
	}
	r := m.Run(context.Background(), "ss", "-tnH", "state", "established", "( sport = :2049 )")
	a.From = "connections to port 2049"
	for _, line := range strings.Split(r.Out, "\n") {
		f := strings.Fields(line)
		if len(f) >= 4 {
			a.Clients = append(a.Clients, Client{Address: f[len(f)-1]})
		}
	}
	return a
}

// TestAnswer is what `test` answers.
type TestAnswer struct {
	Share   string `json:"share"`
	Path    string `json:"path,omitempty"`
	Access  string `json:"access,omitempty"`
	Live    bool   `json:"live"`
	Service string `json:"service"`
	InRange *bool  `json:"in_range,omitempty"`
	OK      bool   `json:"ok"`
	Why     string `json:"why"`
}

// Test says whether one share is served now.
func (m *Machine) Test(share, address string) (TestAnswer, error) {
	if share == "" {
		return TestAnswer{}, fmt.Errorf("name the share")
	}
	a := TestAnswer{Share: share, Service: m.ServiceState()}
	st, err := m.LastStatus()
	if err != nil {
		a.Why = err.Error()
		return a, nil
	}
	for _, r := range st.Refused {
		if r.Name == share {
			a.Path, a.Why = r.Path, r.Why
			return a, nil
		}
	}
	var found *Exported
	for i := range st.Shares {
		if st.Shares[i].Name == share {
			found = &st.Shares[i]
		}
	}
	if found == nil {
		a.Why = fmt.Sprintf("This machine's assignment asks for no share named %s.", share)
		return a, nil
	}
	a.Path, a.Access = found.Path, found.Access
	live, note := m.Live()
	a.Live = note == "" && SharedLive(live, *found)
	var why []string
	if address != "" {
		addr, err1 := netip.ParseAddr(address)
		prefix, err2 := netip.ParsePrefix(st.Range)
		in := err1 == nil && err2 == nil && prefix.Contains(addr)
		a.InRange = &in
		if !in {
			why = append(why, fmt.Sprintf("%s is not inside the private network's range %s.", address, st.Range))
		} else {
			to := false
			for _, g := range found.To {
				if g.Address == address {
					to = true
					a.Access = g.Access
				}
			}
			if !to {
				why = append(why, fmt.Sprintf("The share %s is not exported to %s: no node there is both granted it and wants it.", share, address))
			} else if note != "" || !IsLive(live, found.Path, address) {
				why = append(why, fmt.Sprintf("The kernel does not export %s to %s now.", found.Path, address))
			}
		}
	}
	if len(found.To) == 0 {
		why = append(why, fmt.Sprintf("The share %s is exported to no node.", share))
	} else if !a.Live {
		why = append(why, fmt.Sprintf("The kernel does not export %s to every node it is meant for (%s) now.", found.Path, grantees(found.To)))
	}
	if a.Service != "active" {
		why = append(why, "The NFS server is "+a.Service+".")
	}
	a.OK = len(why) == 0
	if a.OK {
		a.Why = fmt.Sprintf("The share %s (%s) is exported to %s and the NFS server is active.", share, found.Path, grantees(found.To))
	} else {
		a.Why = strings.Join(why, " ")
	}
	return a, nil
}

// Reload has the kernel read the export files again.
func (m *Machine) Reload() (any, error) {
	if _, err := m.AsRoot("exportfs", "-ra"); err != nil {
		return nil, err
	}
	return m.Exports(), nil
}

// AdoptAnswer is what `adopt` answers.
type AdoptAnswer struct {
	Path    string `json:"path"`
	Dataset string `json:"dataset,omitempty"`
	Was     string `json:"sharenfs_was,omitempty"`
	Done    bool   `json:"done"`
	DryRun  bool   `json:"dry_run"`
	Said    string `json:"said"`
}

// Adopt clears one dataset's sharenfs once the mesh's own export of its folder is live.
func (m *Machine) Adopt(p string, confirm bool, why string) (AdoptAnswer, error) {
	if p == "" {
		return AdoptAnswer{}, fmt.Errorf("name the shared folder (path)")
	}
	p = path.Clean(p)
	a := AdoptAnswer{Path: p, DryRun: !confirm}
	sets, note := m.Datasets()
	if note != "" {
		return a, fmt.Errorf("%s", note)
	}
	var ds *Dataset
	for i := range sets {
		if path.Clean(sets[i].Mountpoint) == p {
			ds = &sets[i]
		}
	}
	if ds == nil {
		return a, fmt.Errorf("no ZFS dataset is mounted at %s, so there is no sharenfs to take over", p)
	}
	a.Dataset, a.Was = ds.Name, ds.ShareNFS
	if ds.ShareNFS == "off" || ds.ShareNFS == "-" || ds.ShareNFS == "" {
		return a, fmt.Errorf("the dataset %s already has sharenfs off; nothing to take over", ds.Name)
	}
	st, err := m.LastStatus()
	if err != nil {
		return a, err
	}
	var mine *Exported
	for i := range st.Shares {
		if st.Shares[i].Path == p {
			mine = &st.Shares[i]
		}
	}
	if mine == nil {
		return a, fmt.Errorf("the mesh does not export %s: add it to this machine's nfs-server `shares` setting first, so the folder is never unexported", p)
	}
	if len(mine.To) == 0 {
		return a, fmt.Errorf("the mesh exports %s to no node yet (no node is both granted it and wants it), so taking over would unexport it for everyone", p)
	}
	live, lnote := m.Live()
	if lnote != "" || !SharedLive(live, *mine) {
		return a, fmt.Errorf("the mesh's export of %s to %s is not live in the kernel yet, so taking over would unexport it; wait for the module's process, or call reload", p, grantees(mine.To))
	}
	plan := fmt.Sprintf("set sharenfs=off on the dataset %s (now %q) and run exportfs -ra; %s stays exported as the share %s to %s, every client as %d:%d",
		ds.Name, ds.ShareNFS, p, mine.Name, grantees(mine.To), mine.UID, mine.GID)
	if !confirm {
		a.Said = "Dry run: would " + plan + "."
		return a, nil
	}
	if why == "" {
		return a, fmt.Errorf("say why: adopt is a person's act, and the why is kept with it")
	}
	if _, err := m.AsRoot("zfs", "set", "sharenfs=off", ds.Name); err != nil {
		return a, err
	}
	if _, err := m.AsRoot("exportfs", "-ra"); err != nil {
		return a, fmt.Errorf("sharenfs is off on %s, and exportfs -ra failed: %v", ds.Name, err)
	}
	a.Done = true
	a.Said = fmt.Sprintf("Did: %s. Why: %s", plan, why)
	return a, nil
}

// Health is the module's health, as the node-engine asks it.
type Health struct {
	Healthy bool     `json:"healthy"`
	Why     string   `json:"why"`
	Found   []string `json:"found,omitempty"`
}

// Health judges the shares.
func (m *Machine) Health() Health {
	var found []string
	service := m.ServiceState()
	if service != "active" {
		found = append(found, fmt.Sprintf("The NFS server (%s) is %s.", Unit, service))
	}
	st, err := m.LastStatus()
	if err != nil {
		found = append(found, err.Error()+".")
		return Health{Healthy: false, Why: strings.Join(found, " "), Found: found}
	}
	if age := m.Now().Sub(st.At); age > Stale {
		found = append(found, fmt.Sprintf("The module's process last looked %s ago, at %s.", age.Round(time.Second), st.At.Local().Format("15:04")))
	}
	for _, p := range st.Problems {
		found = append(found, sentence(p))
	}
	for _, r := range st.Refused {
		found = append(found, r.Why)
	}
	if st.ReloadError != "" {
		found = append(found, sentence("the kernel could not read the export files again: "+st.ReloadError))
	}
	live, note := m.Live()
	for _, e := range st.Shares {
		if note == "" && !SharedLive(live, e) {
			found = append(found, fmt.Sprintf("The share %s (%s) is not in the kernel's export table.", e.Name, e.Path))
		}
	}
	sort.Strings(found)
	if len(found) > 0 {
		return Health{Healthy: false, Why: strings.Join(found, " "), Found: found}
	}
	var parts []string
	for _, e := range st.Shares {
		if len(e.To) > 0 {
			parts = append(parts, fmt.Sprintf("%s to %s", e.Name, grantees(e.To)))
		}
	}
	if len(parts) == 0 {
		return Health{Healthy: true, Why: "The NFS server is active, and the mesh exports nothing to any node here."}
	}
	return Health{Healthy: true, Why: fmt.Sprintf("The NFS server is active and exports %s.", strings.Join(parts, "; "))}
}

func sentence(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return s
	}
	s = strings.ToUpper(s[:1]) + s[1:]
	if !strings.HasSuffix(s, ".") {
		s += "."
	}
	return s
}

// grantees says who a share goes to: `ws (10.0.0.3, rw), laptop (10.0.0.4, ro)`.
func grantees(to []Grantee) string {
	var out []string
	for _, g := range to {
		out = append(out, fmt.Sprintf("%s (%s, %s)", g.Node, g.Address, g.Access))
	}
	return strings.Join(out, ", ")
}