mesh-catalog/modules/nfs-server/cmd/nfs-server/process.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/netip"
	"os"
	"path"
	"strings"
	"time"
)

// The module's process, as root: every Every it reads the assignment's shares, renders the export file,
// has the kernel read it again when anything changed or a share is not live, and writes what it saw for
// the tools (novox/hq ADR 0263).

// Every is how often the process looks.
const Every = 30 * time.Second

// Status is the process's last look, as the tools read it.
type Status struct {
	At       time.Time  `json:"at"`
	Range    string     `json:"range"`
	Shares   []Exported `json:"shares"`
	Refused  []Refused  `json:"refused,omitempty"`
	Problems []string   `json:"problems,omitempty"`
	// Said is what does not match between the grants and the wants: a node wanting a share it is not
	// granted, a grant nobody wants, a share exported to no node.
	Said []string `json:"said,omitempty"`
	// Warnings make the module no less healthy but are said in its health: what it could not learn, and
	// how old what it relies on is.
	Warnings []string `json:"warnings,omitempty"`
	// Reloaded is when the process last had the kernel read the export files again, and why.
	Reloaded    *time.Time `json:"reloaded,omitempty"`
	ReloadWhy   string     `json:"reload_why,omitempty"`
	ReloadError string     `json:"reload_error,omitempty"`
}

// Pass is one look: what it did is in the status it answers, which it has written.
func (m *Machine) Pass(ctx context.Context, last *Status) Status {
	s := Status{At: m.Now().UTC()}
	if last != nil {
		s.Reloaded, s.ReloadWhy = last.Reloaded, last.ReloadWhy
	}
	text, err := m.Read(ConfigFile)
	if err != nil {
		s.Problems = append(s.Problems, fmt.Sprintf("the assignment's shares (%s) could not be read: %v; nothing is exported", ConfigFile, err))
	}
	c, problems := ParseConfig(text)
	s.Problems = append(s.Problems, problems...)
	s.Range = c.Range
	roster, note := m.Roster()
	if note != "" {
		s.Problems = append(s.Problems, note)
	}
	wants, note := m.Wants()
	if note != "" {
		s.Said = append(s.Said, note)
	}
	if _, note := m.Backup(); note != "" {
		s.Warnings = append(s.Warnings, note)
	}
	m.setWalk(ctx)
	exports, refused, said := m.Judge(c, roster, wants)
	m.setWalk(nil)
	if ctx.Err() != nil {
		// Stopping: write nothing and reload nothing, so Stopped starts at once.
		s.Problems = append(s.Problems, "the pass was ended by the process stopping; nothing was written")
		return s
	}
	s.Refused = refused
	s.Said = append(s.Said, said...)

	want := ExportFileText(exports)
	have, _ := m.Read(ExportFile)
	reload := ""
	if have != want {
		if err := m.Write(ExportFile, want); err != nil {
			s.Problems = append(s.Problems, fmt.Sprintf("the export file could not be written: %v", err))
		} else {
			reload = "the export file changed"
		}
	}
	live, note := m.Live()
	foreign, _, _ := m.Foreign()
	mine := ParseExports(want, ExportFile)
	if reload == "" && last == nil {
		reload = "the process started, and the kernel's table is made to match the file"
	}
	if reload == "" && last != nil && last.ReloadError != "" {
		reload = "the last reload failed: " + last.ReloadError
	}
	if reload == "" {
		for _, e := range exports {
			if !SharedLive(live, e) {
				reload = fmt.Sprintf("the share %s was not in the kernel's table", e.Name)
				break
			}
		}
	}
	if reload == "" {
		if stray := Stray(live, mine, foreign, c.Range); len(stray) > 0 {
			reload = fmt.Sprintf("the kernel exports %s to %s, which the export file no longer carries", stray[0].Path, stray[0].Client)
		}
	}
	if reload != "" {
		if _, err := m.AsRootWithin(ctx, "exportfs", "-ra"); err != nil {
			s.ReloadError = err.Error()
		} else {
			at := m.Now().UTC()
			s.Reloaded, s.ReloadWhy = &at, reload
		}
		live, note = m.Live()
	}
	if note != "" && len(exports) > 0 {
		s.Problems = append(s.Problems, note)
	}
	for _, e := range Stray(live, mine, foreign, c.Range) {
		s.Problems = append(s.Problems, fmt.Sprintf("the kernel still exports %s to %s, which the module's export file "+
			"no longer carries: a reload has to take it away (exportfs -ra as root)", e.Path, e.Client))
	}
	for i := range exports {
		exports[i].Live = SharedLive(live, exports[i])
	}
	s.Shares = exports
	raw, _ := json.MarshalIndent(s, "", "  ")
	if err := m.Write(StatusFile, string(raw)+"\n"); err != nil {
		s.Problems = append(s.Problems, fmt.Sprintf("the status file could not be written: %v", err))
	}
	return s
}

// ProcessUnit is the service the node-engine runs this process as: the manifest's process name, `.service`.
const ProcessUnit = "nfs-server-exports.service"

// StopTimeout bounds Stopped and the commands it runs: well under the service manager's stop timeout (90 s,
// then SIGKILL), so the export file is gone before the process could be killed (mesh-host #62).
var StopTimeout = 10 * time.Second

// RunExports is the process: a pass now and every Every until the context ends, then Stopped.
func (m *Machine) RunExports(ctx context.Context) error {
	last := m.Pass(ctx, nil)
	t := time.NewTicker(Every)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			m.Stopped()
			return nil
		case <-t.C:
			last = m.Pass(ctx, &last)
		}
	}
}

// LastStatus is the process's last look, or why there is none.
func (m *Machine) LastStatus() (*Status, error) {
	text, err := m.Read(StatusFile)
	if err != nil {
		return nil, fmt.Errorf("the module's process has written no status (%s): %v", StatusFile, err)
	}
	var s Status
	if err := json.Unmarshal([]byte(text), &s); err != nil {
		return nil, fmt.Errorf("the module's status file is not one: %v", err)
	}
	return &s, nil
}

// Roster is the mesh's machines by name, with their private addresses, as the mesh rendered them.
func (m *Machine) Roster() (map[string]string, string) {
	text, err := m.Read(RosterFile)
	if err != nil {
		return map[string]string{}, fmt.Sprintf("the mesh's roster (%s) could not be read: %v; no node's address is known, so nothing is exported", RosterFile, err)
	}
	out := map[string]string{}
	for _, line := range strings.Split(text, "\n") {
		f := strings.Fields(line)
		if len(f) != 2 || strings.HasPrefix(f[0], "#") {
			continue
		}
		out[f[0]] = f[1]
	}
	return out, ""
}

// Wants is every node's wants of this machine, as the tools last took them from the bus. With none taken
// yet, nothing is wanted, and nothing is exported.
func (m *Machine) Wants() (map[string][]Want, string) {
	text, err := m.Read(WantsFile)
	if err != nil {
		return map[string][]Want{}, "no node's wants have been taken from the bus yet, so nothing is exported"
	}
	var w WantsFromBus
	if err := json.Unmarshal([]byte(text), &w); err != nil {
		return map[string][]Want{}, fmt.Sprintf("the wants taken from the bus (%s) could not be read: %v; nothing is exported", WantsFile, err)
	}
	if w.Wants == nil {
		w.Wants = map[string][]Want{}
	}
	return w.Wants, ""
}

// SharedLive says whether the kernel exports a share to every node it is meant for.
func SharedLive(live []Entry, e Exported) bool {
	for _, g := range e.To {
		if !IsLive(live, e.Path, g.Address) {
			return false
		}
	}
	return true
}

// Stopped is the process ending. When the module was unassigned (see unassigned) the module's export file
// goes and the kernel lets the shares go. A plain restart, a push or the machine shutting down leaves the
// config file and the unit enabled, and the exports stay.
func (m *Machine) Stopped() {
	ctx, cancel := context.WithTimeout(context.Background(), StopTimeout)
	defer cancel()
	if !m.unassigned(ctx) {
		return
	}
	// The file first: removing it needs no command, so a hanging sudo or exportfs cannot keep it.
	if err := m.unexport(); err != nil {
		loud("the module was unassigned and its export file is still there: " + err.Error() +
			"; remove " + ExportFile + " and run exportfs -ra as root")
		return
	}
	fmt.Fprintln(os.Stderr, "nfs-server: unassigned: the export file "+ExportFile+" is removed")
	if _, err := m.AsRootWithin(ctx, "exportfs", "-ra"); err != nil {
		loud("the module was unassigned and its export file is removed, but the kernel may still export its " +
			"shares until exportfs -ra runs as root: " + err.Error())
		return
	}
	fmt.Fprintln(os.Stderr, "nfs-server: unassigned: the kernel let the shares go (exportfs -ra)")
}

// unassigned says whether this stop is the module being taken away: its config file is gone (the node-engine
// removes resources in reverse of their record, and the config is recorded after this process), or this
// process's own unit is disabled. The node-engine stops a process it removes with `systemctl disable
// --now`, which disables the unit before it stops it, while a restart, a push or a shutdown leaves it
// enabled — so the second sign holds even when a failed first apply recorded the process after its config
// (mesh-host #62) and the config is still there.
func (m *Machine) unassigned(ctx context.Context) bool {
	if _, err := m.Read(ConfigFile); os.IsNotExist(err) {
		return true
	}
	r := m.Run(ctx, "systemctl", "is-enabled", ProcessUnit)
	return strings.TrimSpace(r.Out) == "disabled"
}

// loud says, on the process's log where the service manager's journal keeps it, what an unassignment left
// undone, so a person finds it rather than an export that outlived its module.
func loud(what string) {
	fmt.Fprintln(os.Stderr, "nfs-server: UNASSIGNED, NOT CLEAN: "+what)
}

// Stray is what the kernel exports that is the module's and that its file no longer carries: an entry to a
// client inside the private network's range — the only clients the module ever exports to — that neither
// the module's file nor any other export file (/etc/exports, ZFS's own) has. What another file exports is
// not the module's, whatever its client.
func Stray(live, mine, foreign []Entry, rng string) []Entry {
	prefix, err := netip.ParsePrefix(rng)
	if err != nil {
		return nil
	}
	has := map[string]bool{}
	for _, e := range append(append([]Entry{}, mine...), foreign...) {
		has[path.Clean(e.Path)+" "+e.Client] = true
	}
	var out []Entry
	for _, e := range live {
		if has[path.Clean(e.Path)+" "+e.Client] || !insideRange(e.Client, prefix) {
			continue
		}
		out = append(out, e)
	}
	return out
}

// insideRange says whether a client — one address, or a network — lies wholly inside a range.
func insideRange(client string, prefix netip.Prefix) bool {
	if a, err := netip.ParseAddr(client); err == nil {
		return prefix.Contains(a)
	}
	if p, err := netip.ParsePrefix(client); err == nil {
		return p.Bits() >= prefix.Bits() && prefix.Contains(p.Addr())
	}
	return false
}