mounts: clear a failed mount of its own whose source was away, and the failure of a unit it takes over, so neither is said as the machine's own failed unit (hq ADR 0263 rules 4 and 6)

modules/mounts/README.md

@@ -42,6 +42,14 @@ an autofs mount, so nothing writes into the empty directory under it.
 | an SMB source | `//<host>/<share>`, `Type=cifs` | `credentials=<file>,iocharset=utf8,_netdev,nofail,uid=…,gid=…` | 10 |
 | a disk | `/dev/disk/by-uuid/<uuid>`; the mount `BindsTo=` the disk's device | `nofail` | 10 |
 
+A mount of the module's own that failed because its source was not there — an occasional source that is
+off, or a node's share whose server is away, tried when someone opened the mount point — is cleared
+(`systemctl reset-failed`) by `apply` within five seconds, and so is the failure a unit of the same name
+carried before the module wrote it (the unit fstab's generator makes until the line is adopted). The module
+says such a share as `absent` or `unreachable`; left failed, the unit would also be said as a failed unit of
+the machine's own, with a notification each time it is touched. A mount that failed while its source answers
+is left failed: that is a fault, and the share is said `failed`.
+
 A share is read-only when the setting says `:ro` or the sharing machine exports it read-only. A
 read-write share is mounted hard, so a write is never dropped while the server is away.
 

modules/mounts/cmd/mounts/apply.go

@@ -33,6 +33,11 @@ import (
 // LookEvery is how often the process looks.
 const LookEvery = 30 * time.Second
 
+// ClearEvery is how often the acting process clears a failed mount of its own whose source does not
+// answer (clearFailed). Shorter than the node-engine's two looks of 15 s, so a mount that failed because
+// its source is away is never read as a failed unit of the machine's own.
+const ClearEvery = 5 * time.Second
+
 // ProbeTimeout bounds one try of a source.
 const ProbeTimeout = 2 * time.Second
 
@@ -89,6 +94,9 @@ type Applier struct {
 	Dial   func(addr string, timeout time.Duration) error
 	Now    func() time.Time
 	Lookup func(account string) (int, int, error)
+	// written is every pair the last pass wrote and armed, by its mount unit: what clearFailed looks at
+	// between passes. The acting process's own memory; nothing else reads it.
+	written map[string]Pair
 }
 
 // NewApplier is the process on this machine.
@@ -326,6 +334,7 @@ func (a *Applier) Look(ctx context.Context) Status {
 	own := a.ownUnits()
 	mounted := ParseMountinfo(func() string { t, _ := a.M.read("/proc/self/mountinfo"); return t }())
 	changed := false
+	fresh := map[string]bool{} // the mount units this pass wrote anew
 	var written []Pair
 	for _, p := range pairs {
 		ours := own[p.MountUnit] && own[p.AutoUnit]
@@ -364,6 +373,7 @@ func (a *Applier) Look(ctx context.Context) Status {
 		}
 		if c1 || c2 {
 			changed = true
+			fresh[p.MountUnit] = true
 			st.Changed = append(st.Changed, "the units of "+p.Name)
 		}
 		// A missing mount point is made; one that is there is left exactly as it is. An automount point
@@ -448,6 +458,23 @@ func (a *Applier) Look(ctx context.Context) Status {
 			}
 		}
 	}
+	if a.Act {
+		// A mount unit this pass wrote anew over one that had failed — the unit fstab's generator made for
+		// the same mount point, before the line was adopted — carries a failure that was not the module's
+		// pair's. It is cleared, so the pair starts armed and is judged on its own mounts (ADR 0263 rule 4).
+		for _, p := range written {
+			if fresh[p.MountUnit] && states[p.MountUnit]["ActiveState"] == "failed" {
+				if _, err := a.M.Run(ctx, "systemctl", "reset-failed", p.MountUnit); err == nil {
+					st.Changed = append(st.Changed, "cleared "+p.MountUnit+", which had failed before the module wrote it")
+				}
+			}
+		}
+		a.written = map[string]Pair{}
+		for _, p := range written {
+			a.written[p.MountUnit] = p
+		}
+		st.Changed = append(st.Changed, a.clearFailed(ctx)...)
+	}
 	if !a.Judge {
 		a.finish(&st, notSet)
 		return st
@@ -569,17 +596,70 @@ func (a *Applier) writeStatus(st *Status) {
 	}
 }
 
-// RunApply looks every LookEvery until the context ends, saying on stderr what it changed.
+// clearFailed clears every failed mount of the module's own pairs whose source does not answer now: an
+// occasional source that is away, or a node's share whose server is. Such a mount failed because its
+// source was not there, which the module already says as the share's state (absent, or unreachable, and
+// for a share that should always be there its health after five minutes, ADR 0263 rule 8); left failed,
+// the unit would be said a second time as a failed unit of the machine's own, with a notification each
+// time it is touched — what rules 4 and 6 promise never happens. A mount that failed while its source
+// answers is left failed: that is a fault, and the share is said failed until a person looks.
+func (a *Applier) clearFailed(ctx context.Context) []string {
+	if len(a.written) == 0 {
+		return nil
+	}
+	var units []string
+	for unit := range a.written {
+		units = append(units, unit)
+	}
+	sort.Strings(units)
+	states := a.unitStates(ctx, units)
+	var cleared []string
+	for _, unit := range units {
+		if states[unit]["ActiveState"] != "failed" {
+			continue
+		}
+		p := a.written[unit]
+		if a.sourceAnswers(p) {
+			continue
+		}
+		if _, err := a.M.Run(ctx, "systemctl", "reset-failed", unit); err == nil {
+			cleared = append(cleared, "cleared "+unit+": "+p.Name+" was not there to mount")
+		}
+	}
+	return cleared
+}
+
+// sourceAnswers tries a pair's source the way the judging process does, without touching its mount point.
+func (a *Applier) sourceAnswers(p Pair) bool {
+	if p.Kind == "disk" {
+		return a.M.present(p.Probe)
+	}
+	return a.Dial != nil && a.Dial(p.Probe, ProbeTimeout) == nil
+}
+
+// RunApply looks every LookEvery until the context ends, saying on stderr what it changed. Between looks
+// the acting process clears, every ClearEvery, a mount of its own that failed because its source was away.
 func RunApply(ctx context.Context, a *Applier) {
 	for {
 		st := a.Look(ctx)
 		for _, c := range st.Changed {
 			fmt.Fprintln(os.Stderr, "mounts:", c)
 		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(LookEvery):
+		next := time.After(LookEvery)
+	between:
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-next:
+				break between
+			case <-time.After(ClearEvery):
+				if a.Act {
+					for _, c := range a.clearFailed(ctx) {
+						fmt.Fprintln(os.Stderr, "mounts:", c)
+					}
+				}
+			}
 		}
 	}
 }

modules/mounts/cmd/mounts/failed_test.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+// A mount the module took over, or one whose source was away when it was touched, is never left as a
+// failed unit: the node-engine would say it as a failed unit of the machine's own, with a notification each
+// time (hq ADR 0263 rules 4 and 6). A mount that failed while its source answers stays failed: a fault.
+
+func failedLook(t *testing.T, f *fakeSystem) map[string]ShareStatus {
+	t.Helper()
+	st := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)).Look(context.Background())
+	out := map[string]ShareStatus{}
+	for _, s := range st.Shares {
+		out[s.Name] = s
+	}
+	return out
+}
+
+func TestTheFailureOfTheUnitTheModuleTakesOverIsCleared(t *testing.T) {
+	f := newSystem(t)
+	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
+	f.write(ExportsFile, exportsAnswer(t, library))
+	f.reachable["192.0.2.10:2049"] = true
+	// fstab's generator made srv-library.mount for the same mount point, and it timed out at boot.
+	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
+
+	states := failedLook(t, f)
+
+	if !f.called("systemctl reset-failed srv-library.mount") {
+		t.Fatalf("the generator's failure is cleared once the module's unit is written: %q", f.calls)
+	}
+	if s := states["library"]; s.State != StateArmed {
+		t.Errorf("the share is judged on its own mounts, not on the generator's: %+v", s)
+	}
+}
+
+func TestAMountThatFailedBecauseItsSourceWasAwayIsCleared(t *testing.T) {
+	f := newSystem(t)
+	f.write(ConfigFile, "shares=library=/srv/library\nsources=games=smb://192.0.2.76/Share@/mnt/games\n"+
+		"server=homeserver\nat=192.0.2.10\nnode=workstation\naccount=operator\n"+
+		"secret=/var/lib/mesh/mounts/smb-credentials.secret\nstate=/var/lib/mesh/mounts\n")
+	f.write(ExportsFile, exportsAnswer(t, library))
+	f.write("/var/lib/mesh/mounts/smb-credentials.secret", "[games]\nusername=root\npassword="+testPassword+"\n")
+	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
+	a.Look(context.Background()) // writes and arms both pairs; neither source answers
+	f.calls = nil
+
+	// Someone opened both mount points: the gaming device is off and the server is away, so both mounts failed.
+	f.units["mnt-games.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
+	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
+	cleared := a.clearFailed(context.Background())
+
+	for _, unit := range []string{"mnt-games.mount", "srv-library.mount"} {
+		if !f.called("systemctl reset-failed " + unit) {
+			t.Errorf("%s failed because its source was away and is cleared: %q", unit, f.calls)
+		}
+	}
+	if len(cleared) != 2 || !strings.Contains(strings.Join(cleared, "; "), "games was not there to mount") {
+		t.Errorf("each clearing is said: %q", cleared)
+	}
+	states := failedLook(t, f)
+	if states["games"].State != StateAbsent || states["library"].State != StateUnreachable {
+		t.Errorf("the module still says each as absent or unreachable: %+v", states)
+	}
+}
+
+func TestAMountThatFailedWhileItsSourceAnswersStaysFailed(t *testing.T) {
+	f := newSystem(t)
+	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
+	f.write(ExportsFile, exportsAnswer(t, library))
+	f.reachable["192.0.2.10:2049"] = true
+	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
+	a.Look(context.Background())
+	f.calls = nil
+
+	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "exit-code"}
+	if cleared := a.clearFailed(context.Background()); len(cleared) != 0 || f.called("systemctl reset-failed") {
+		t.Fatalf("a mount refused by a server that answers is a fault and is left failed: %q %q", cleared, f.calls)
+	}
+	if s := failedLook(t, f)["library"]; s.State != StateFailed {
+		t.Errorf("it is said failed: %+v", s)
+	}
+}
+
+func TestOnlyTheActingProcessClears(t *testing.T) {
+	f := newSystem(t)
+	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
+	f.write(ExportsFile, exportsAnswer(t, library))
+	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
+	a := f.applier(time.Now())
+	a.Act = false
+	a.Look(context.Background())
+	if f.called("systemctl reset-failed") {
+		t.Errorf("the judging process changes nothing: %q", f.calls)
+	}
+}

modules/mounts/cmd/mounts/write_test.go

@@ -77,6 +77,11 @@ func (f *fakeSystem) run(_ context.Context, name string, args ...string) (string
 	case "enable":
 		u := args[len(args)-1]
 		f.units[u] = map[string]string{"ActiveState": "active", "UnitFileState": "enabled"}
+	case "reset-failed":
+		u := args[len(args)-1]
+		if f.units[u]["ActiveState"] == "failed" {
+			f.units[u]["ActiveState"], f.units[u]["Result"] = "inactive", "success"
+		}
 	}
 	return "", nil
 }