mesh-catalog/modules/mounts/cmd/mounts/apply.go

package main

// The module's two long-running processes, run by the node-engine as root (novox/hq ADR 0263, to-be 52 §2),
// over one pass of the same code every 30 seconds:
//
//   - `mounts apply` acts: it writes or removes the module's own unit pairs and the SMB logins, makes a
//     missing mount point and arms the automounts, and leaves what it could not do in AppliedFile. The
//     service manager judges it by its unit;
//   - `mounts watch` judges: it tries each source without touching a mount point and leaves what it found
//     in StatusFile, which the health check `mounts_health`, the tools and the bundle read.
//
// Neither reaches the bus: the bundle reads what the sharing machine exports and leaves it in ExportsFile,
// and puts StatusFile in the module's state.
//
// What it never does: write, stop or remove a unit file without the module's marker; chown or chmod a
// mount point (it makes one that is missing, and nothing else); remove a pair because the sharing machine
// did not answer — a pair goes only when the settings stop naming it.

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// LookEvery is how often the process looks.
const LookEvery = 30 * time.Second

// ProbeTimeout bounds one try of a source.
const ProbeTimeout = 2 * time.Second

// The states of a share (to-be 52 §2).
const (
	StateArmed       = "armed"       // the automount waits for the first use
	StateMounted     = "mounted"     // mounted now
	StateAbsent      = "absent"      // an occasional source that is not there: a state, not a fault
	StateUnreachable = "unreachable" // should be there and is not
	StateFailed      = "failed"      // tried and refused, or not set up; why says which
)

// ExportsAnswer is what the bundle left from the sharing machine's `node-nfs-server.exports`.
type ExportsAnswer struct {
	Server string        `json:"server"`
	At     string        `json:"at,omitempty"`
	Asked  string        `json:"asked"`
	Shares []ServerShare `json:"shares"`
	// Error is why the last ask got no answer; the shares are then the last answer's.
	Error    string `json:"error,omitempty"`
	Answered string `json:"answered,omitempty"`
}

// ShareStatus is one share or source as the process found it.
type ShareStatus struct {
	Name       string `json:"name"`
	Kind       string `json:"kind"`
	Occasional bool   `json:"occasional"`
	Mountpoint string `json:"mountpoint"`
	What       string `json:"what,omitempty"`
	Probe      string `json:"probe,omitempty"`
	State      string `json:"state"`
	Since      string `json:"since"`
	Why        string `json:"why,omitempty"`
	MountUnit  string `json:"mount_unit,omitempty"`
	AutoUnit   string `json:"automount_unit,omitempty"`
}

// Status is what one look found.
type Status struct {
	Looked   string        `json:"looked"`
	Server   string        `json:"server,omitempty"`
	Shares   []ShareStatus `json:"shares"`
	Problems []string      `json:"problems,omitempty"`
	Changed  []string      `json:"changed,omitempty"`
}

// Applier is the process, over one machine. Act writes and arms; Judge tries and says. A test does both.
type Applier struct {
	M     *Machine
	Act   bool
	Judge bool
	// Dial tries a host:port; Now is the clock; Lookup finds an account's uid and gid.
	Dial   func(addr string, timeout time.Duration) error
	Now    func() time.Time
	Lookup func(account string) (int, int, error)
}

// NewApplier is the process on this machine.
func NewApplier() *Applier {
	return &Applier{
		M: Here(),
		Dial: func(addr string, timeout time.Duration) error {
			c, err := net.DialTimeout("tcp", addr, timeout)
			if err == nil {
				c.Close()
			}
			return err
		},
		Now: time.Now,
		Lookup: func(account string) (int, int, error) {
			u, err := user.Lookup(account)
			if err != nil {
				return -1, -1, err
			}
			uid, _ := strconv.Atoi(u.Uid)
			gid, _ := strconv.Atoi(u.Gid)
			return uid, gid, nil
		},
	}
}

func (a *Applier) path(p string) string { return filepath.Join(a.M.Root, p) }

// writeIfChanged writes a file through a temporary beside it, only when its content differs.
func (a *Applier) writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
	full := a.path(p)
	if old, err := os.ReadFile(full); err == nil && string(old) == content {
		if info, err := os.Stat(full); err == nil && info.Mode().Perm() == mode {
			return false, nil
		}
	}
	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
		return false, err
	}
	tmp := full + ".mesh-new"
	if err := os.WriteFile(tmp, []byte(content), mode); err != nil {
		return false, err
	}
	if err := os.Chmod(tmp, mode); err != nil {
		os.Remove(tmp)
		return false, err
	}
	return true, os.Rename(tmp, full)
}

// ownUnits are the unit files in UnitDir that carry the module's marker, by name.
func (a *Applier) ownUnits() map[string]bool {
	out := map[string]bool{}
	for _, pattern := range []string{"*.mount", "*.automount"} {
		for _, p := range a.M.glob(filepath.Join(UnitDir, pattern)) {
			if IsOwnUnit(a.M, p) {
				out[filepath.Base(p)] = true
			}
		}
	}
	return out
}

// IsOwnUnit says whether a unit file opens with the module's marker.
func IsOwnUnit(m *Machine, p string) bool {
	text, err := m.read(p)
	return err == nil && strings.HasPrefix(text, Marker)
}

// unitStates are the service manager's states for some units: ActiveState, SubState, Result and
// UnitFileState, by unit.
func (a *Applier) unitStates(ctx context.Context, units []string) map[string]map[string]string {
	out := map[string]map[string]string{}
	if len(units) == 0 {
		return out
	}
	args := append([]string{"show", "-p", "Id,ActiveState,SubState,Result,UnitFileState", "--no-pager"}, units...)
	shown, _ := a.M.Run(ctx, "systemctl", args...)
	for _, block := range strings.Split(shown, "\n\n") {
		kv := map[string]string{}
		for _, line := range strings.Split(block, "\n") {
			if k, v, ok := strings.Cut(line, "="); ok {
				kv[k] = v
			}
		}
		if kv["Id"] != "" {
			out[kv["Id"]] = kv
		}
	}
	return out
}

func readJSON[T any](m *Machine, p string) (T, error) {
	var v T
	text, err := m.read(p)
	if err != nil {
		return v, err
	}
	return v, json.Unmarshal([]byte(text), &v)
}

// Look is one pass: the units written as asked, the automounts armed, and each share found.
func (a *Applier) Look(ctx context.Context) Status {
	now := a.Now()
	st := Status{Looked: now.Format(time.RFC3339), Shares: []ShareStatus{}}
	previous, _ := readJSON[Status](a.M, StatusFile)
	since := map[string]ShareStatus{}
	for _, s := range previous.Shares {
		since[s.Name] = s
	}

	text, err := a.M.read(ConfigFile)
	if err != nil {
		st.Problems = append(st.Problems, unreadable(ConfigFile, err)+"; nothing is written or removed until the mesh places it")
		a.finish(&st, nil)
		return st
	}
	cfg := ParseConfig(text)
	st.Server = cfg.ServerNode()
	st.Problems = append(st.Problems, cfg.Problems...)

	exports, eerr := readJSON[ExportsAnswer](a.M, ExportsFile)
	byName := map[string]ServerShare{}
	for _, e := range exports.Shares {
		byName[e.Name] = e
	}

	var pairs []Pair
	notSet := map[string]ShareStatus{} // a share named but not set up, with why
	for _, w := range cfg.Shares {
		e, ok := byName[w.Name]
		switch {
		case cfg.At == "":
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
				Why: "no machine shares nfs-share for this one: the binding gives no address"}
		case !ok && eerr != nil:
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateUnreachable,
				Why: "what the sharing machine " + st.Server + " exports has not been read from the bus yet"}
		case !ok:
			why := "the sharing machine " + exports.Server + " does not export a share named " + w.Name
			if exports.Error != "" {
				why = "what the sharing machine " + st.Server + " exports could not be read from the bus: " + exports.Error
			}
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed, Why: why}
		case !strings.HasPrefix(e.Path, "/") || strings.ContainsAny(e.Path, " \t\n\\\"'"):
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
				Why: fmt.Sprintf("the sharing machine gave the path %q, which the module does not mount", e.Path)}
		default:
			pairs = append(pairs, NFSPair(w, cfg.At, e))
		}
	}

	var creds map[string]SMBCredential
	needCreds := false
	for _, s := range cfg.Sources {
		needCreds = needCreds || s.Kind == "smb"
	}
	if needCreds {
		secret, err := a.M.read(cfg.Secret)
		if cfg.Secret == "" || err != nil {
			st.Problems = append(st.Problems, "the module's secret smb-credentials is not on this machine")
		} else {
			var p []string
			creds, p = ParseSMBCredentials(secret)
			for _, x := range p {
				st.Problems = append(st.Problems, "smb-credentials: "+x)
			}
		}
	}
	uid, gid := -1, -1
	if cfg.Account != "" && a.Lookup != nil {
		if u, g, err := a.Lookup(cfg.Account); err == nil {
			uid, gid = u, g
		}
	}
	for _, s := range cfg.Sources {
		switch s.Kind {
		case "disk":
			pairs = append(pairs, DiskPair(s))
		case "smb":
			c, ok := creds[s.Name]
			if !ok || cfg.State == "" {
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateFailed,
					Why: fmt.Sprintf("its login is not given: put a [%s] section in the module's secret with `secret accept <node> mounts smb-credentials --from <file>`", s.Name)}
				continue
			}
			file := filepath.Join(cfg.State, "smb-"+s.Name+".credentials")
			changed, err := false, error(nil)
			if a.Act {
				changed, err = a.writeIfChanged(file, c.CredentialsFile(), 0o600)
			}
			if err != nil {
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateFailed,
					Why: "its credentials file could not be written: " + err.Error()}
				continue
			}
			if changed {
				st.Changed = append(st.Changed, "the login of "+s.Name)
			}
			pairs = append(pairs, SMBPair(s, file, uid, gid))
		}
	}

	// Every mount point the settings still name, set up or not: a pair is removed only when its mount
	// point is no longer named, never because an answer is missing.
	named := map[string]bool{}
	for _, w := range cfg.Shares {
		named[EscapePath(w.Mountpoint)] = true
	}
	for _, s := range cfg.Sources {
		named[EscapePath(s.Mountpoint)] = true
	}

	own := a.ownUnits()
	mounted := ParseMountinfo(func() string { t, _ := a.M.read("/proc/self/mountinfo"); return t }())
	changed := false
	var written []Pair
	for _, p := range pairs {
		if err := a.foreign(p, own); err != nil {
			notSet[p.Name] = ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
				State: StateFailed, Why: err.Error()}
			continue
		}
		if !a.Act {
			written = append(written, p)
			continue
		}
		c1, err1 := a.writeIfChanged(filepath.Join(UnitDir, p.MountUnit), p.Mount, 0o644)
		c2, err2 := a.writeIfChanged(filepath.Join(UnitDir, p.AutoUnit), p.Automount, 0o644)
		if err1 != nil || err2 != nil {
			notSet[p.Name] = ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
				State: StateFailed, Why: fmt.Sprintf("its units could not be written: %v %v", err1, err2)}
			continue
		}
		if c1 || c2 {
			changed = true
			st.Changed = append(st.Changed, "the units of "+p.Name)
		}
		// A missing mount point is made; one that is there is left exactly as it is. An automount point
		// is in mountinfo and is never looked at, so a server that is away cannot hold this look.
		if _, there := mounted[p.Mountpoint]; !there {
			if _, err := os.Lstat(a.path(p.Mountpoint)); os.IsNotExist(err) {
				if err := os.MkdirAll(a.path(p.Mountpoint), 0o755); err == nil {
					st.Changed = append(st.Changed, "made the mount point "+p.Mountpoint)
				}
			}
		}
		written = append(written, p)
	}

	// The module's own pairs whose mount point nothing names any more: stopped, disabled and removed.
	var gone []string
	for unit := range own {
		base := strings.TrimSuffix(strings.TrimSuffix(unit, ".automount"), ".mount")
		if !named[base] && a.Act {
			gone = append(gone, unit)
		}
	}
	sort.Strings(gone)
	for _, unit := range gone {
		if strings.HasSuffix(unit, ".automount") {
			a.M.Run(ctx, "systemctl", "disable", "--now", unit)
		} else {
			a.M.Run(ctx, "systemctl", "stop", unit)
		}
	}
	for _, unit := range gone {
		if err := os.Remove(a.path(filepath.Join(UnitDir, unit))); err == nil {
			changed = true
			st.Changed = append(st.Changed, "removed "+unit+", which nothing names any more")
		}
	}
	if changed {
		if _, err := a.M.Run(ctx, "systemctl", "daemon-reload"); err != nil {
			st.Problems = append(st.Problems, "the service manager did not reload: "+err.Error())
		}
	}

	// Arm every automount that is not.
	var units []string
	for _, p := range written {
		units = append(units, p.MountUnit, p.AutoUnit)
	}
	states := a.unitStates(ctx, units)
	for _, p := range written {
		auto := states[p.AutoUnit]
		if a.Act && (auto["UnitFileState"] != "enabled" || auto["ActiveState"] != "active") {
			if _, err := a.M.Run(ctx, "systemctl", "enable", "--now", p.AutoUnit); err != nil {
				st.Problems = append(st.Problems, fmt.Sprintf("%s was not armed: %v", p.AutoUnit, err))
			} else {
				st.Changed = append(st.Changed, "armed "+p.AutoUnit)
			}
		}
	}
	if !a.Judge {
		a.finish(&st, notSet)
		return st
	}
	if len(written) > 0 {
		states = a.unitStates(ctx, units)
	}
	// What the acting process could not do, which only it knows: a unit or a login it could not write.
	if applied, err := readJSON[Applied](a.M, AppliedFile); err == nil && !a.Act {
		st.Problems = append(st.Problems, applied.Problems...)
		for _, s := range applied.NotSet {
			if _, said := notSet[s.Name]; !said {
				notSet[s.Name] = s
			}
		}
	}

	for _, p := range written {
		if _, failed := notSet[p.Name]; failed {
			continue
		}
		s := ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
			Probe: p.Probe, MountUnit: p.MountUnit, AutoUnit: p.AutoUnit}
		s.State, s.Why = a.judge(p, states, mounted)
		st.Shares = append(st.Shares, s)
	}
	for _, s := range notSet {
		st.Shares = append(st.Shares, s)
	}
	sort.Slice(st.Shares, func(i, j int) bool { return st.Shares[i].Name < st.Shares[j].Name })
	for i := range st.Shares {
		s := &st.Shares[i]
		if prev, ok := since[s.Name]; ok && prev.State == s.State && prev.Since != "" {
			s.Since = prev.Since
		} else {
			s.Since = st.Looked
		}
	}
	a.finish(&st, notSet)
	return st
}

// AppliedFile is what the acting process could not do, for the judging one.
const AppliedFile = RunDir + "/applied.json"

// Applied is the acting process's last pass.
type Applied struct {
	Looked   string        `json:"looked"`
	Problems []string      `json:"problems,omitempty"`
	NotSet   []ShareStatus `json:"not_set,omitempty"`
	Changed  []string      `json:"changed,omitempty"`
}

// finish leaves what this pass found: the acting process's account, the judging one's status.
func (a *Applier) finish(st *Status, notSet map[string]ShareStatus) {
	if a.Act && !a.Judge {
		ap := Applied{Looked: st.Looked, Problems: st.Problems, Changed: st.Changed}
		for _, s := range notSet {
			ap.NotSet = append(ap.NotSet, s)
		}
		sort.Slice(ap.NotSet, func(i, j int) bool { return ap.NotSet[i].Name < ap.NotSet[j].Name })
		raw, _ := json.MarshalIndent(ap, "", "  ")
		if _, err := a.writeIfChanged(AppliedFile, string(raw)+"\n", 0o644); err != nil {
			fmt.Fprintln(os.Stderr, "mounts: what was applied was not written:", err)
		}
	}
	if a.Judge {
		a.writeStatus(st)
	}
}

// foreign refuses a pair whose unit file exists and is not the module's.
func (a *Applier) foreign(p Pair, own map[string]bool) error {
	for _, unit := range []string{p.MountUnit, p.AutoUnit} {
		if own[unit] {
			continue
		}
		if _, err := os.Lstat(a.path(filepath.Join(UnitDir, unit))); err == nil {
			return fmt.Errorf("%s already has a unit file that is not the module's (%s); it is left as it is, and this share is not mounted here", p.Mountpoint, filepath.Join(UnitDir, unit))
		}
	}
	return nil
}

// judge is one pair's state. The source is tried first, without touching the mount point.
func (a *Applier) judge(p Pair, states map[string]map[string]string, mounted map[string]Mounted) (string, string) {
	there := true
	var tried error
	if p.Kind == "disk" {
		if !a.M.present(p.Probe) {
			there, tried = false, fmt.Errorf("the disk %s is not plugged in", p.Probe)
		}
	} else if err := a.Dial(p.Probe, ProbeTimeout); err != nil {
		there, tried = false, err
	}
	mnt, auto := states[p.MountUnit], states[p.AutoUnit]
	cur, isMounted := mounted[p.Mountpoint]
	isMounted = isMounted && cur.Type != "autofs"
	switch {
	case !there && p.Occasional:
		return StateAbsent, "not present: " + tried.Error()
	case !there:
		return StateUnreachable, fmt.Sprintf("%s does not answer: %v", p.Probe, tried)
	case isMounted:
		return StateMounted, ""
	case mnt["ActiveState"] == "failed":
		return StateFailed, fmt.Sprintf("its last mount failed (%s); see the journal of %s", orDefault(mnt["Result"], "failed"), p.MountUnit)
	case auto["ActiveState"] == "active":
		return StateArmed, ""
	default:
		return StateFailed, fmt.Sprintf("its automount %s is not armed (%s)", p.AutoUnit, orDefault(auto["ActiveState"], "unknown"))
	}
}

func (a *Applier) writeStatus(st *Status) {
	raw, _ := json.MarshalIndent(st, "", "  ")
	if _, err := a.writeIfChanged(StatusFile, string(raw)+"\n", 0o644); err != nil {
		fmt.Fprintln(os.Stderr, "mounts: the status was not written:", err)
	}
}

// RunApply looks every LookEvery until the context ends, saying on stderr what it changed.
func RunApply(ctx context.Context, a *Applier) {
	for {
		st := a.Look(ctx)
		for _, c := range st.Changed {
			fmt.Fprintln(os.Stderr, "mounts:", c)
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(LookEvery):
		}
	}
}