mounts: clear a failed mount of its own whose source was away, and the failure of a unit it takes over, so neither is said as the machine's own failed unit (hq ADR 0263 rules 4 and 6)

modules/mounts/cmd/mounts/apply.go

@@ -33,6 +33,11 @@ import (
 // LookEvery is how often the process looks.
 const LookEvery = 30 * time.Second
 
+// ClearEvery is how often the acting process clears a failed mount of its own whose source does not
+// answer (clearFailed). Shorter than the node-engine's two looks of 15 s, so a mount that failed because
+// its source is away is never read as a failed unit of the machine's own.
+const ClearEvery = 5 * time.Second
+
 // ProbeTimeout bounds one try of a source.
 const ProbeTimeout = 2 * time.Second
 
@@ -89,6 +94,9 @@ type Applier struct {
 	Dial   func(addr string, timeout time.Duration) error
 	Now    func() time.Time
 	Lookup func(account string) (int, int, error)
+	// written is every pair the last pass wrote and armed, by its mount unit: what clearFailed looks at
+	// between passes. The acting process's own memory; nothing else reads it.
+	written map[string]Pair
 }
 
 // NewApplier is the process on this machine.
@@ -326,6 +334,7 @@ func (a *Applier) Look(ctx context.Context) Status {
 	own := a.ownUnits()
 	mounted := ParseMountinfo(func() string { t, _ := a.M.read("/proc/self/mountinfo"); return t }())
 	changed := false
+	fresh := map[string]bool{} // the mount units this pass wrote anew
 	var written []Pair
 	for _, p := range pairs {
 		ours := own[p.MountUnit] && own[p.AutoUnit]
@@ -364,6 +373,7 @@ func (a *Applier) Look(ctx context.Context) Status {
 		}
 		if c1 || c2 {
 			changed = true
+			fresh[p.MountUnit] = true
 			st.Changed = append(st.Changed, "the units of "+p.Name)
 		}
 		// A missing mount point is made; one that is there is left exactly as it is. An automount point
@@ -448,6 +458,23 @@ func (a *Applier) Look(ctx context.Context) Status {
 			}
 		}
 	}
+	if a.Act {
+		// A mount unit this pass wrote anew over one that had failed — the unit fstab's generator made for
+		// the same mount point, before the line was adopted — carries a failure that was not the module's
+		// pair's. It is cleared, so the pair starts armed and is judged on its own mounts (ADR 0263 rule 4).
+		for _, p := range written {
+			if fresh[p.MountUnit] && states[p.MountUnit]["ActiveState"] == "failed" {
+				if _, err := a.M.Run(ctx, "systemctl", "reset-failed", p.MountUnit); err == nil {
+					st.Changed = append(st.Changed, "cleared "+p.MountUnit+", which had failed before the module wrote it")
+				}
+			}
+		}
+		a.written = map[string]Pair{}
+		for _, p := range written {
+			a.written[p.MountUnit] = p
+		}
+		st.Changed = append(st.Changed, a.clearFailed(ctx)...)
+	}
 	if !a.Judge {
 		a.finish(&st, notSet)
 		return st
@@ -569,17 +596,70 @@ func (a *Applier) writeStatus(st *Status) {
 	}
 }
 
-// RunApply looks every LookEvery until the context ends, saying on stderr what it changed.
+// clearFailed clears every failed mount of the module's own pairs whose source does not answer now: an
+// occasional source that is away, or a node's share whose server is. Such a mount failed because its
+// source was not there, which the module already says as the share's state (absent, or unreachable, and
+// for a share that should always be there its health after five minutes, ADR 0263 rule 8); left failed,
+// the unit would be said a second time as a failed unit of the machine's own, with a notification each
+// time it is touched — what rules 4 and 6 promise never happens. A mount that failed while its source
+// answers is left failed: that is a fault, and the share is said failed until a person looks.
+func (a *Applier) clearFailed(ctx context.Context) []string {
+	if len(a.written) == 0 {
+		return nil
+	}
+	var units []string
+	for unit := range a.written {
+		units = append(units, unit)
+	}
+	sort.Strings(units)
+	states := a.unitStates(ctx, units)
+	var cleared []string
+	for _, unit := range units {
+		if states[unit]["ActiveState"] != "failed" {
+			continue
+		}
+		p := a.written[unit]
+		if a.sourceAnswers(p) {
+			continue
+		}
+		if _, err := a.M.Run(ctx, "systemctl", "reset-failed", unit); err == nil {
+			cleared = append(cleared, "cleared "+unit+": "+p.Name+" was not there to mount")
+		}
+	}
+	return cleared
+}
+
+// sourceAnswers tries a pair's source the way the judging process does, without touching its mount point.
+func (a *Applier) sourceAnswers(p Pair) bool {
+	if p.Kind == "disk" {
+		return a.M.present(p.Probe)
+	}
+	return a.Dial != nil && a.Dial(p.Probe, ProbeTimeout) == nil
+}
+
+// RunApply looks every LookEvery until the context ends, saying on stderr what it changed. Between looks
+// the acting process clears, every ClearEvery, a mount of its own that failed because its source was away.
 func RunApply(ctx context.Context, a *Applier) {
 	for {
 		st := a.Look(ctx)
 		for _, c := range st.Changed {
 			fmt.Fprintln(os.Stderr, "mounts:", c)
 		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(LookEvery):
+		next := time.After(LookEvery)
+	between:
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-next:
+				break between
+			case <-time.After(ClearEvery):
+				if a.Act {
+					for _, c := range a.clearFailed(ctx) {
+						fmt.Fprintln(os.Stderr, "mounts:", c)
+					}
+				}
+			}
 		}
 	}
 }