mesh-catalog/modules/mounts/cmd/mounts/apply.go

package main

// The module's two long-running processes, run by the node-engine as root (novox/hq ADR 0263, to-be 52 §2),
// over one pass of the same code every 30 seconds:
//
//   - `mounts apply` acts: it writes or removes the module's own unit pairs and the SMB logins, makes a
//     missing mount point and arms the automounts, and leaves what it could not do in AppliedFile. The
//     service manager judges it by its unit;
//   - `mounts watch` judges: it tries each source without touching a mount point and leaves what it found
//     in StatusFile, which the health check `mounts_health`, the tools and the bundle read.
//
// Neither reaches the bus: the bundle reads what the sharing machine exports and leaves it in ExportsFile,
// and puts StatusFile in the module's state.
//
// What it never does: write, stop or remove a unit file without the module's marker; chown or chmod a
// mount point (it makes one that is missing, and nothing else); remove a pair because the sharing machine
// did not answer — a pair goes only when the settings stop naming it.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
)

// LookEvery is how often the process looks.
const LookEvery = 30 * time.Second

// ClearEvery is how often the acting process clears a failed mount of its own whose source does not
// answer (clearFailed). Shorter than the node-engine's two looks of 15 s, so a mount that failed because
// its source is away is never read as a failed unit of the machine's own.
const ClearEvery = 5 * time.Second

// ProbeTimeout bounds one try of a source.
const ProbeTimeout = 2 * time.Second

// The states of a share (to-be 52 §2).
const (
	StateArmed       = "armed"       // the automount waits for the first use
	StateMounted     = "mounted"     // mounted now
	StateAbsent      = "absent"      // an occasional source that is not there: a state, not a fault
	StateUnreachable = "unreachable" // should be there and is not
	StateFailed      = "failed"      // tried and refused, or not set up; why says which
	// StateWaiting is a source that cannot be set up until the operator gives its username or password
	// (novox/hq ADR 0283): not a fault, said with what it waits for.
	StateWaiting = "waiting"
)

// ExportsAnswer is what the bundle left from the sharing machine's `node-nfs-server.exports`.
type ExportsAnswer struct {
	Server string        `json:"server"`
	At     string        `json:"at,omitempty"`
	Asked  string        `json:"asked"`
	Shares []ServerShare `json:"shares"`
	// Error is why the last ask got no answer; the shares are then the last answer's.
	Error    string `json:"error,omitempty"`
	Answered string `json:"answered,omitempty"`
}

// ShareStatus is one share or source as the process found it.
type ShareStatus struct {
	Name       string `json:"name"`
	Kind       string `json:"kind"`
	Occasional bool   `json:"occasional"`
	Mountpoint string `json:"mountpoint"`
	What       string `json:"what,omitempty"`
	Probe      string `json:"probe,omitempty"`
	State      string `json:"state"`
	Since      string `json:"since"`
	Why        string `json:"why,omitempty"`
	MountUnit  string `json:"mount_unit,omitempty"`
	AutoUnit   string `json:"automount_unit,omitempty"`
	// Waits is what a waiting source waits for: its username in the setting sources, or its secret
	// smb-password-<source>.
	Waits []health.Wait `json:"waits,omitempty"`
}

// Status is what one look found.
type Status struct {
	Looked   string        `json:"looked"`
	Server   string        `json:"server,omitempty"`
	Shares   []ShareStatus `json:"shares"`
	Problems []string      `json:"problems,omitempty"`
	Changed  []string      `json:"changed,omitempty"`
}

// Applier is the process, over one machine. Act writes and arms; Judge tries and says. A test does both.
type Applier struct {
	M     *Machine
	Act   bool
	Judge bool
	// Dial tries a host:port; Now is the clock; Lookup finds an account's uid and gid.
	Dial   func(addr string, timeout time.Duration) error
	Now    func() time.Time
	Lookup func(account string) (int, int, error)
	// written is every pair the last pass wrote and armed, by its mount unit: what clearFailed looks at
	// between passes. The acting process's own memory; nothing else reads it.
	written map[string]Pair
	// LookInterval and ClearInterval are LookEvery and ClearEvery, or a test's own when set.
	LookInterval, ClearInterval time.Duration
}

// NewApplier is the process on this machine.
func NewApplier() *Applier {
	return &Applier{
		M: Here(),
		Dial: func(addr string, timeout time.Duration) error {
			c, err := net.DialTimeout("tcp", addr, timeout)
			if err == nil {
				c.Close()
			}
			return err
		},
		Now: time.Now,
		Lookup: func(account string) (int, int, error) {
			u, err := user.Lookup(account)
			if err != nil {
				return -1, -1, err
			}
			uid, _ := strconv.Atoi(u.Uid)
			gid, _ := strconv.Atoi(u.Gid)
			return uid, gid, nil
		},
	}
}

func (a *Applier) path(p string) string { return filepath.Join(a.M.Root, p) }

// writeIfChanged writes a file through a temporary beside it, only when its content differs.
func (a *Applier) writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
	full := a.path(p)
	if old, err := os.ReadFile(full); err == nil && string(old) == content {
		if info, err := os.Stat(full); err == nil && info.Mode().Perm() == mode {
			return false, nil
		}
	}
	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
		return false, err
	}
	tmp := full + ".mesh-new"
	if err := os.WriteFile(tmp, []byte(content), mode); err != nil {
		return false, err
	}
	if err := os.Chmod(tmp, mode); err != nil {
		os.Remove(tmp)
		return false, err
	}
	return true, os.Rename(tmp, full)
}

// ownUnits are the unit files in UnitDir that carry the module's marker, by name.
func (a *Applier) ownUnits() map[string]bool {
	out := map[string]bool{}
	for _, pattern := range []string{"*.mount", "*.automount"} {
		for _, p := range a.M.glob(filepath.Join(UnitDir, pattern)) {
			if IsOwnUnit(a.M, p) {
				out[filepath.Base(p)] = true
			}
		}
	}
	return out
}

// IsOwnUnit says whether a unit file opens with the module's marker.
func IsOwnUnit(m *Machine, p string) bool {
	text, err := m.read(p)
	return err == nil && strings.HasPrefix(text, Marker)
}

// unitStates are the service manager's states for some units: ActiveState, SubState, Result and
// UnitFileState, by unit.
func (a *Applier) unitStates(ctx context.Context, units []string) map[string]map[string]string {
	out := map[string]map[string]string{}
	if len(units) == 0 {
		return out
	}
	args := append([]string{"show", "-p", "Id,ActiveState,SubState,Result,UnitFileState", "--no-pager"}, units...)
	shown, _ := a.M.Run(ctx, "systemctl", args...)
	for _, block := range strings.Split(shown, "\n\n") {
		kv := map[string]string{}
		for _, line := range strings.Split(block, "\n") {
			if k, v, ok := strings.Cut(line, "="); ok {
				kv[k] = v
			}
		}
		if kv["Id"] != "" {
			out[kv["Id"]] = kv
		}
	}
	return out
}

func readJSON[T any](m *Machine, p string) (T, error) {
	var v T
	text, err := m.read(p)
	if err != nil {
		return v, err
	}
	return v, json.Unmarshal([]byte(text), &v)
}

// Look is one pass: the units written as asked, the automounts armed, and each share found.
func (a *Applier) Look(ctx context.Context) Status {
	now := a.Now()
	st := Status{Looked: now.Format(time.RFC3339), Shares: []ShareStatus{}}
	previous, _ := readJSON[Status](a.M, StatusFile)
	since := map[string]ShareStatus{}
	for _, s := range previous.Shares {
		since[s.Name] = s
	}

	text, err := a.M.read(ConfigFile)
	if err != nil {
		st.Problems = append(st.Problems, unreadable(ConfigFile, err)+"; nothing is written or removed until the mesh places it")
		a.finish(&st, nil)
		return st
	}
	cfg := ParseConfig(text)
	st.Server = cfg.ServerNode()
	st.Problems = append(st.Problems, cfg.Problems...)

	// Where the mesh keeps data on this machine (finding 4d). Without it nothing new is set up: a pair the
	// module already has stays, and one it does not have waits until the list is read.
	data, derr := readJSON[DataPaths](a.M, DataFile)
	haveData := derr == nil && data.Paths != nil
	if !haveData {
		why := "it has not been left by the module's tools yet"
		if derr == nil && data.Error != "" {
			why = data.Error
		}
		st.Problems = append(st.Problems, "where the mesh keeps data on this machine is not known ("+why+"); nothing new is set up until it is")
	}

	exports, eerr := readJSON[ExportsAnswer](a.M, ExportsFile)
	byName := map[string]ServerShare{}
	for _, e := range exports.Shares {
		byName[e.Name] = e
	}

	var pairs []Pair
	notSet := map[string]ShareStatus{} // a share named but not set up, with why
	for _, w := range cfg.Shares {
		e, ok := byName[w.Name]
		switch {
		case cfg.At == "":
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
				Why: "no machine shares nfs-share for this one: the binding gives no address"}
		case cfg.Node != "" && st.Server == cfg.Node:
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
				Why: "this machine shares it itself (" + cfg.Node + "): a share is never mounted onto the machine that serves it"}
		case !ok && eerr != nil:
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateUnreachable,
				Why: "what the sharing machine " + st.Server + " exports has not been read from the bus yet"}
		case !ok:
			why := "the sharing machine " + exports.Server + " does not export a share named " + w.Name
			if exports.Error != "" {
				why = "what the sharing machine " + st.Server + " exports could not be read from the bus: " + exports.Error
			}
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed, Why: why}
		case !strings.HasPrefix(e.Path, "/") || strings.ContainsAny(e.Path, " \t\n\\\"'"):
			notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
				Why: fmt.Sprintf("the sharing machine gave the path %q, which the module does not mount", e.Path)}
		default:
			mine, exported := e.ForNode(cfg.Node)
			if !exported {
				notSet[w.Name] = ShareStatus{Name: w.Name, Kind: "nfs", Mountpoint: w.Mountpoint, State: StateFailed,
					Why: "the sharing machine " + st.Server + " does not export " + w.Name + " to this machine: its grants do not name " + cfg.Node}
				continue
			}
			pairs = append(pairs, NFSPair(w, cfg.At, mine))
		}
	}

	uid, gid := -1, -1
	if cfg.Account != "" && a.Lookup != nil {
		if u, g, err := a.Lookup(cfg.Account); err == nil {
			uid, gid = u, g
		}
	}
	for _, s := range cfg.Sources {
		switch s.Kind {
		case "disk":
			pairs = append(pairs, DiskPair(s))
		case "smb":
			c, waits, why := a.login(cfg, s)
			switch {
			case why != "":
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateFailed, Why: why}
				continue
			case len(waits) > 0:
				var gives []string
				for _, w := range waits {
					gives = append(gives, w.What+" ("+w.Names()+")")
				}
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateWaiting,
					Why: "it waits for the operator to give " + strings.Join(gives, " and "), Waits: waits}
				continue
			case cfg.State == "":
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateFailed,
					Why: "the module's state directory is not named in its configuration, so no credentials file can be written"}
				continue
			}
			file := filepath.Join(cfg.State, "smb-"+s.Name+".credentials")
			changed, err := false, error(nil)
			if a.Act {
				changed, err = a.writeIfChanged(file, c.CredentialsFile(), 0o600)
			}
			if err != nil {
				notSet[s.Name] = ShareStatus{Name: s.Name, Kind: "smb", Occasional: true, Mountpoint: s.Mountpoint, State: StateFailed,
					Why: "its credentials file could not be written: " + err.Error()}
				continue
			}
			if changed {
				st.Changed = append(st.Changed, "the login of "+s.Name)
			}
			pairs = append(pairs, SMBPair(s, file, uid, gid))
		}
	}

	// Every mount point the settings still name, set up or not: a pair is removed only when its mount
	// point is no longer named, never because an answer is missing.
	named := map[string]bool{}
	for _, w := range cfg.Shares {
		named[EscapePath(w.Mountpoint)] = true
	}
	for _, s := range cfg.Sources {
		named[EscapePath(s.Mountpoint)] = true
	}

	own := a.ownUnits()
	mounted := ParseMountinfo(func() string { t, _ := a.M.read("/proc/self/mountinfo"); return t }())
	changed := false
	// The mount units this pass wrote where no unit file of the module's was before: a unit of the same
	// name that was there — the one fstab's generator makes until the line is adopted — is taken over.
	takenOver := map[string]bool{}
	var written []Pair
	for _, p := range pairs {
		ours := own[p.MountUnit] && own[p.AutoUnit]
		refuse := func(why string) {
			notSet[p.Name] = ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
				State: StateFailed, Why: why}
		}
		if haveData {
			if err := nearData(p.Mountpoint, data.Paths); err != nil {
				refuse(err.Error())
				continue
			}
		} else if !ours {
			refuse("where the mesh keeps data on this machine is not known yet (the controller's data verb), so a new mount point is not set up")
			continue
		}
		if err := a.M.mountpointFree(p, own, mounted); err != nil {
			refuse(err.Error())
			continue
		}
		if err := a.foreign(p, own); err != nil {
			notSet[p.Name] = ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
				State: StateFailed, Why: err.Error()}
			continue
		}
		if !a.Act {
			written = append(written, p)
			continue
		}
		c1, err1 := a.writeIfChanged(filepath.Join(UnitDir, p.MountUnit), p.Mount, 0o644)
		c2, err2 := a.writeIfChanged(filepath.Join(UnitDir, p.AutoUnit), p.Automount, 0o644)
		if err1 != nil || err2 != nil {
			notSet[p.Name] = ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
				State: StateFailed, Why: fmt.Sprintf("its units could not be written: %v %v", err1, err2)}
			continue
		}
		if c1 && !own[p.MountUnit] {
			takenOver[p.MountUnit] = true
		}
		if c1 || c2 {
			changed = true
			st.Changed = append(st.Changed, "the units of "+p.Name)
		}
		// A missing mount point is made; one that is there is left exactly as it is. An automount point
		// is in mountinfo and is never looked at, so a server that is away cannot hold this look.
		if _, there := mounted[p.Mountpoint]; !there {
			if _, err := os.Lstat(a.path(p.Mountpoint)); os.IsNotExist(err) {
				if err := os.MkdirAll(a.path(p.Mountpoint), 0o755); err == nil {
					st.Changed = append(st.Changed, "made the mount point "+p.Mountpoint)
				}
			}
		}
		written = append(written, p)
	}

	// The module's own pairs whose mount point nothing names any more: stopped, disabled and removed —
	// unless an entry of the settings could not be read, when nobody can tell which mount point it named
	// (finding 8), or what is mounted there is not the pair's own source (finding 4c).
	var gone []string
	if len(cfg.Unreadable) > 0 {
		if a.Act {
			st.Problems = append(st.Problems, "nothing is removed while an entry of shares or sources cannot be read: "+strings.Join(cfg.Unreadable, "; "))
		}
	} else {
		kept := map[string]bool{}
		for unit := range own {
			base := strings.TrimSuffix(strings.TrimSuffix(unit, ".automount"), ".mount")
			if named[base] || !a.Act || kept[base] {
				continue
			}
			text, _ := a.M.read(filepath.Join(UnitDir, base+".mount"))
			what, where := unitField(text, "What"), unitField(text, "Where")
			if where != "" {
				if err := a.M.notOwnSource(kindOf(what, unitField(text, "Type")), what, where, mounted); err != nil {
					kept[base] = true
					st.Problems = append(st.Problems, fmt.Sprintf("the pair %s is no longer named and is kept, and nothing is stopped: %v", base, err))
					continue
				}
			}
			gone = append(gone, unit)
		}
		var still []string
		for _, unit := range gone {
			if !kept[strings.TrimSuffix(strings.TrimSuffix(unit, ".automount"), ".mount")] {
				still = append(still, unit)
			}
		}
		gone = still
	}
	sort.Strings(gone)
	for _, unit := range gone {
		if strings.HasSuffix(unit, ".automount") {
			a.M.Run(ctx, "systemctl", "disable", "--now", unit)
		} else {
			a.M.Run(ctx, "systemctl", "stop", unit)
		}
	}
	for _, unit := range gone {
		if err := os.Remove(a.path(filepath.Join(UnitDir, unit))); err == nil {
			changed = true
			st.Changed = append(st.Changed, "removed "+unit+", which nothing names any more")
		}
	}
	if changed {
		if _, err := a.M.Run(ctx, "systemctl", "daemon-reload"); err != nil {
			st.Problems = append(st.Problems, "the service manager did not reload: "+err.Error())
		}
	}

	// Arm every automount that is not.
	var units []string
	for _, p := range written {
		units = append(units, p.MountUnit, p.AutoUnit)
	}
	states := a.unitStates(ctx, units)
	for _, p := range written {
		auto := states[p.AutoUnit]
		if a.Act && (auto["UnitFileState"] != "enabled" || auto["ActiveState"] != "active") {
			if _, err := a.M.Run(ctx, "systemctl", "enable", "--now", p.AutoUnit); err != nil {
				st.Problems = append(st.Problems, fmt.Sprintf("%s was not armed: %v", p.AutoUnit, err))
			} else {
				st.Changed = append(st.Changed, "armed "+p.AutoUnit)
			}
		}
	}
	if a.Act {
		// A mount unit the module wrote for the first time, over a unit of the same name that had already
		// failed — the one fstab's generator makes until the line is adopted — carries a failure that is not
		// the module's pair's. It is cleared, so the pair starts armed and is judged on its own mounts (ADR
		// 0263 rule 4). A pair the module already had and rewrote keeps its failure: that one is its own.
		for _, p := range written {
			if takenOver[p.MountUnit] && states[p.MountUnit]["ActiveState"] == "failed" {
				if _, err := a.M.Run(ctx, "systemctl", "reset-failed", p.MountUnit); err == nil {
					st.Changed = append(st.Changed, "cleared "+p.MountUnit+", which had failed before the module wrote it")
				}
			}
		}
		a.written = map[string]Pair{}
		for _, p := range written {
			a.written[p.MountUnit] = p
		}
		st.Changed = append(st.Changed, a.clearFailed(ctx)...)
	}
	if !a.Judge {
		a.finish(&st, notSet)
		return st
	}
	if len(written) > 0 {
		states = a.unitStates(ctx, units)
	}
	// What the acting process could not do, which only it knows: a unit or a login it could not write.
	if applied, err := readJSON[Applied](a.M, AppliedFile); err == nil && !a.Act {
		st.Problems = append(st.Problems, applied.Problems...)
		for _, s := range applied.NotSet {
			if _, said := notSet[s.Name]; !said {
				notSet[s.Name] = s
			}
		}
	}

	for _, p := range written {
		if _, failed := notSet[p.Name]; failed {
			continue
		}
		s := ShareStatus{Name: p.Name, Kind: p.Kind, Occasional: p.Occasional, Mountpoint: p.Mountpoint, What: p.What,
			Probe: p.Probe, MountUnit: p.MountUnit, AutoUnit: p.AutoUnit}
		s.State, s.Why = a.judge(p, states, mounted)
		st.Shares = append(st.Shares, s)
	}
	for _, s := range notSet {
		st.Shares = append(st.Shares, s)
	}
	sort.Slice(st.Shares, func(i, j int) bool { return st.Shares[i].Name < st.Shares[j].Name })
	for i := range st.Shares {
		s := &st.Shares[i]
		if prev, ok := since[s.Name]; ok && prev.State == s.State && prev.Since != "" {
			s.Since = prev.Since
		} else {
			s.Since = st.Looked
		}
	}
	a.finish(&st, notSet)
	return st
}

// AppliedFile is what the acting process could not do, for the judging one.
const AppliedFile = RunDir + "/applied.json"

// Applied is the acting process's last pass.
type Applied struct {
	Looked   string        `json:"looked"`
	Problems []string      `json:"problems,omitempty"`
	NotSet   []ShareStatus `json:"not_set,omitempty"`
	Changed  []string      `json:"changed,omitempty"`
}

// finish leaves what this pass found: the acting process's account, the judging one's status.
func (a *Applier) finish(st *Status, notSet map[string]ShareStatus) {
	if a.Act && !a.Judge {
		ap := Applied{Looked: st.Looked, Problems: st.Problems, Changed: st.Changed}
		for _, s := range notSet {
			ap.NotSet = append(ap.NotSet, s)
		}
		sort.Slice(ap.NotSet, func(i, j int) bool { return ap.NotSet[i].Name < ap.NotSet[j].Name })
		raw, _ := json.MarshalIndent(ap, "", "  ")
		if _, err := a.writeIfChanged(AppliedFile, string(raw)+"\n", 0o644); err != nil {
			fmt.Fprintln(os.Stderr, "mounts: what was applied was not written:", err)
		}
	}
	if a.Judge {
		a.writeStatus(st)
	}
}

// foreign refuses a pair whose unit file exists and is not the module's.
func (a *Applier) foreign(p Pair, own map[string]bool) error {
	for _, unit := range []string{p.MountUnit, p.AutoUnit} {
		// A transient unit of the same name — made by `systemd-mount` or `mount` on the command line — is
		// someone's mount now, whatever the module's own file says.
		transient := filepath.Join(TransientDir, unit)
		if _, err := os.Lstat(a.path(transient)); err == nil {
			return fmt.Errorf("%s has a transient unit that is not the module's (%s); it is left as it is, and this share is not mounted here", p.Mountpoint, transient)
		}
		if own[unit] {
			continue
		}
		if _, err := os.Lstat(a.path(filepath.Join(UnitDir, unit))); err == nil {
			return fmt.Errorf("%s already has a unit file that is not the module's (%s); it is left as it is, and this share is not mounted here", p.Mountpoint, filepath.Join(UnitDir, unit))
		}
	}
	return nil
}

// TransientDir is where the service manager keeps the units made at run time rather than from a file.
const TransientDir = "/run/systemd/transient"

// judge is one pair's state. The source is tried first, without touching the mount point.
func (a *Applier) judge(p Pair, states map[string]map[string]string, mounted map[string]Mounted) (string, string) {
	there := true
	var tried error
	if p.Kind == "disk" {
		if !a.M.present(p.Probe) {
			there, tried = false, fmt.Errorf("the disk %s is not plugged in", p.Probe)
		}
	} else if err := a.Dial(p.Probe, ProbeTimeout); err != nil {
		there, tried = false, err
	}
	mnt, auto := states[p.MountUnit], states[p.AutoUnit]
	cur, isMounted := mounted[p.Mountpoint]
	isMounted = isMounted && !cur.IsAutomount()
	switch {
	case !there && p.Occasional:
		return StateAbsent, "not present: " + tried.Error()
	case !there:
		return StateUnreachable, fmt.Sprintf("%s does not answer: %v", p.Probe, tried)
	case isMounted:
		return StateMounted, ""
	case mnt["ActiveState"] == "failed":
		return StateFailed, fmt.Sprintf("its last mount failed (%s); see the journal of %s", orDefault(mnt["Result"], "failed"), p.MountUnit)
	case auto["ActiveState"] == "active":
		return StateArmed, ""
	default:
		return StateFailed, fmt.Sprintf("its automount %s is not armed (%s)", p.AutoUnit, orDefault(auto["ActiveState"], "unknown"))
	}
}

func (a *Applier) writeStatus(st *Status) {
	raw, _ := json.MarshalIndent(st, "", "  ")
	if _, err := a.writeIfChanged(StatusFile, string(raw)+"\n", 0o644); err != nil {
		fmt.Fprintln(os.Stderr, "mounts: the status was not written:", err)
	}
}

// clearFailed clears every failed mount of the module's own pairs whose source does not answer now: an
// occasional source that is away, or a node's share whose server is. Such a mount failed because its
// source was not there, which the module already says as the share's state (absent, or unreachable, and
// for a share that should always be there its health after five minutes, ADR 0263 rule 8); left failed,
// the unit would be said a second time as a failed unit of the machine's own, with a notification each
// time it is touched — what rules 4 and 6 promise never happens. A mount that failed while its source
// answers is left failed: that is a fault, and the share is said failed until a person looks.
func (a *Applier) clearFailed(ctx context.Context) []string {
	if len(a.written) == 0 {
		return nil
	}
	var mounts, units []string
	for unit, p := range a.written {
		mounts = append(mounts, unit)
		units = append(units, unit, p.AutoUnit)
	}
	sort.Strings(mounts)
	states := a.unitStates(ctx, units)
	var cleared []string
	for _, unit := range mounts {
		p := a.written[unit]
		mountFailed := states[unit]["ActiveState"] == "failed"
		// After repeated failed mounts the automount gives up too (mount-start-limit-hit) and stops
		// answering its point: cleared and armed again, or the share stays dead until the next pass.
		autoFailed := states[p.AutoUnit]["ActiveState"] == "failed"
		if !mountFailed && !autoFailed {
			continue
		}
		if a.sourceAnswers(p) {
			continue
		}
		if mountFailed {
			if _, err := a.M.Run(ctx, "systemctl", "reset-failed", unit); err == nil {
				cleared = append(cleared, "cleared "+unit+": "+p.Name+" was not there to mount")
			}
		}
		if autoFailed {
			if _, err := a.M.Run(ctx, "systemctl", "reset-failed", p.AutoUnit); err == nil {
				if _, err := a.M.Run(ctx, "systemctl", "start", p.AutoUnit); err == nil {
					cleared = append(cleared, "cleared and armed "+p.AutoUnit+" again ("+orDefault(states[p.AutoUnit]["Result"], "failed")+"): "+p.Name+" was not there to mount")
				}
			}
		}
	}
	return cleared
}

// sourceAnswers tries a pair's source the way the judging process does, without touching its mount point.
func (a *Applier) sourceAnswers(p Pair) bool {
	if p.Kind == "disk" {
		return a.M.present(p.Probe)
	}
	return a.Dial != nil && a.Dial(p.Probe, ProbeTimeout) == nil
}

// RunApply looks every LookEvery until the context ends, saying on stderr what it changed. Between looks
// the acting process clears, every ClearEvery, a mount of its own that failed because its source was away.
func RunApply(ctx context.Context, a *Applier) {
	for {
		st := a.Look(ctx)
		for _, c := range st.Changed {
			fmt.Fprintln(os.Stderr, "mounts:", c)
		}
		next := time.After(orInterval(a.LookInterval, LookEvery))
	between:
		for {
			select {
			case <-ctx.Done():
				return
			case <-next:
				break between
			case <-time.After(orInterval(a.ClearInterval, ClearEvery)):
				if a.Act {
					for _, c := range a.clearFailed(ctx) {
						fmt.Fprintln(os.Stderr, "mounts:", c)
					}
				}
			}
		}
	}
}

func orInterval(d, otherwise time.Duration) time.Duration {
	if d > 0 {
		return d
	}
	return otherwise
}

// login is an SMB source's login, or what it waits for, or why it cannot be read. The password is read as
// one line from the source's own secret; it is never said, and neither is anything the file holds.
func (a *Applier) login(cfg Config, s Source) (SMBCredential, []health.Wait, string) {
	var waits []health.Wait
	source, user := s.Name, s.User
	part := "the source " + source
	if user == "" {
		waits = append(waits, health.Wait{Part: part, Setting: "sources",
			What: "the username of the source " + source + ", in its entry of sources"})
	}
	// Never a relative path: without the directory the mesh places the passwords in, nothing is read.
	if !filepath.IsAbs(cfg.Passwords) {
		return SMBCredential{}, nil, "the module's configuration names no directory for its passwords, so " +
			PasswordSecret(source) + " cannot be read"
	}
	var password string
	raw, err := a.M.read(PasswordFile(cfg.Passwords, source))
	switch {
	case errors.Is(err, fs.ErrNotExist):
		waits = append(waits, health.Wait{Part: part, Secret: PasswordSecret(source), What: "the password of the source " + source})
	case err != nil:
		return SMBCredential{}, nil, "its password " + PasswordSecret(source) + " could not be read: " + bare(err)
	default:
		pw, given, perr := ReadPassword(raw)
		if perr != nil {
			return SMBCredential{}, nil, "its password " + PasswordSecret(source) + " " + perr.Error() + ": give it again"
		}
		if !given {
			waits = append(waits, health.Wait{Part: part, Secret: PasswordSecret(source), What: "the password of the source " + source})
		}
		password = pw
	}
	if len(waits) > 0 {
		return SMBCredential{}, waits, ""
	}
	return SMBCredential{Username: user, Password: password}, nil, ""
}

// bare is an error without the path it names: the reason alone.
func bare(err error) string {
	var pe *fs.PathError
	if errors.As(err, &pe) {
		return pe.Err.Error()
	}
	return err.Error()
}