diff --git a/modules/mounts/README.md b/modules/mounts/README.md index 0e1a1911..92cdc4bb 100644 --- a/modules/mounts/README.md +++ b/modules/mounts/README.md @@ -42,6 +42,14 @@ an autofs mount, so nothing writes into the empty directory under it. | an SMB source | `///`, `Type=cifs` | `credentials=,iocharset=utf8,_netdev,nofail,uid=…,gid=…` | 10 | | a disk | `/dev/disk/by-uuid/`; the mount `BindsTo=` the disk's device | `nofail` | 10 | +A mount of the module's own that failed because its source was not there — an occasional source that is +off, or a node's share whose server is away, tried when someone opened the mount point — is cleared +(`systemctl reset-failed`) by `apply` within five seconds, and so is the failure a unit of the same name +carried before the module wrote it (the unit fstab's generator makes until the line is adopted). The module +says such a share as `absent` or `unreachable`; left failed, the unit would also be said as a failed unit of +the machine's own, with a notification each time it is touched. A mount that failed while its source answers +is left failed: that is a fault, and the share is said `failed`. + A share is read-only when the setting says `:ro` or the sharing machine exports it read-only. A read-write share is mounted hard, so a write is never dropped while the server is away. diff --git a/modules/mounts/cmd/mounts/apply.go b/modules/mounts/cmd/mounts/apply.go index c2df2630..19910c01 100644 --- a/modules/mounts/cmd/mounts/apply.go +++ b/modules/mounts/cmd/mounts/apply.go @@ -33,6 +33,11 @@ import ( // LookEvery is how often the process looks. const LookEvery = 30 * time.Second +// ClearEvery is how often the acting process clears a failed mount of its own whose source does not +// answer (clearFailed). Shorter than the node-engine's two looks of 15 s, so a mount that failed because +// its source is away is never read as a failed unit of the machine's own. +const ClearEvery = 5 * time.Second + // ProbeTimeout bounds one try of a source. const ProbeTimeout = 2 * time.Second @@ -89,6 +94,9 @@ type Applier struct { Dial func(addr string, timeout time.Duration) error Now func() time.Time Lookup func(account string) (int, int, error) + // written is every pair the last pass wrote and armed, by its mount unit: what clearFailed looks at + // between passes. The acting process's own memory; nothing else reads it. + written map[string]Pair } // NewApplier is the process on this machine. @@ -326,6 +334,7 @@ func (a *Applier) Look(ctx context.Context) Status { own := a.ownUnits() mounted := ParseMountinfo(func() string { t, _ := a.M.read("/proc/self/mountinfo"); return t }()) changed := false + fresh := map[string]bool{} // the mount units this pass wrote anew var written []Pair for _, p := range pairs { ours := own[p.MountUnit] && own[p.AutoUnit] @@ -364,6 +373,7 @@ func (a *Applier) Look(ctx context.Context) Status { } if c1 || c2 { changed = true + fresh[p.MountUnit] = true st.Changed = append(st.Changed, "the units of "+p.Name) } // A missing mount point is made; one that is there is left exactly as it is. An automount point @@ -448,6 +458,23 @@ func (a *Applier) Look(ctx context.Context) Status { } } } + if a.Act { + // A mount unit this pass wrote anew over one that had failed — the unit fstab's generator made for + // the same mount point, before the line was adopted — carries a failure that was not the module's + // pair's. It is cleared, so the pair starts armed and is judged on its own mounts (ADR 0263 rule 4). + for _, p := range written { + if fresh[p.MountUnit] && states[p.MountUnit]["ActiveState"] == "failed" { + if _, err := a.M.Run(ctx, "systemctl", "reset-failed", p.MountUnit); err == nil { + st.Changed = append(st.Changed, "cleared "+p.MountUnit+", which had failed before the module wrote it") + } + } + } + a.written = map[string]Pair{} + for _, p := range written { + a.written[p.MountUnit] = p + } + st.Changed = append(st.Changed, a.clearFailed(ctx)...) + } if !a.Judge { a.finish(&st, notSet) return st @@ -569,17 +596,70 @@ func (a *Applier) writeStatus(st *Status) { } } -// RunApply looks every LookEvery until the context ends, saying on stderr what it changed. +// clearFailed clears every failed mount of the module's own pairs whose source does not answer now: an +// occasional source that is away, or a node's share whose server is. Such a mount failed because its +// source was not there, which the module already says as the share's state (absent, or unreachable, and +// for a share that should always be there its health after five minutes, ADR 0263 rule 8); left failed, +// the unit would be said a second time as a failed unit of the machine's own, with a notification each +// time it is touched — what rules 4 and 6 promise never happens. A mount that failed while its source +// answers is left failed: that is a fault, and the share is said failed until a person looks. +func (a *Applier) clearFailed(ctx context.Context) []string { + if len(a.written) == 0 { + return nil + } + var units []string + for unit := range a.written { + units = append(units, unit) + } + sort.Strings(units) + states := a.unitStates(ctx, units) + var cleared []string + for _, unit := range units { + if states[unit]["ActiveState"] != "failed" { + continue + } + p := a.written[unit] + if a.sourceAnswers(p) { + continue + } + if _, err := a.M.Run(ctx, "systemctl", "reset-failed", unit); err == nil { + cleared = append(cleared, "cleared "+unit+": "+p.Name+" was not there to mount") + } + } + return cleared +} + +// sourceAnswers tries a pair's source the way the judging process does, without touching its mount point. +func (a *Applier) sourceAnswers(p Pair) bool { + if p.Kind == "disk" { + return a.M.present(p.Probe) + } + return a.Dial != nil && a.Dial(p.Probe, ProbeTimeout) == nil +} + +// RunApply looks every LookEvery until the context ends, saying on stderr what it changed. Between looks +// the acting process clears, every ClearEvery, a mount of its own that failed because its source was away. func RunApply(ctx context.Context, a *Applier) { for { st := a.Look(ctx) for _, c := range st.Changed { fmt.Fprintln(os.Stderr, "mounts:", c) } - select { - case <-ctx.Done(): - return - case <-time.After(LookEvery): + next := time.After(LookEvery) + between: + for { + select { + case <-ctx.Done(): + return + case <-next: + break between + case <-time.After(ClearEvery): + if a.Act { + for _, c := range a.clearFailed(ctx) { + fmt.Fprintln(os.Stderr, "mounts:", c) + } + } + } } } } diff --git a/modules/mounts/cmd/mounts/failed_test.go b/modules/mounts/cmd/mounts/failed_test.go new file mode 100644 index 00000000..2762d5c3 --- /dev/null +++ b/modules/mounts/cmd/mounts/failed_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" +) + +// A mount the module took over, or one whose source was away when it was touched, is never left as a +// failed unit: the node-engine would say it as a failed unit of the machine's own, with a notification each +// time (hq ADR 0263 rules 4 and 6). A mount that failed while its source answers stays failed: a fault. + +func failedLook(t *testing.T, f *fakeSystem) map[string]ShareStatus { + t.Helper() + st := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)).Look(context.Background()) + out := map[string]ShareStatus{} + for _, s := range st.Shares { + out[s.Name] = s + } + return out +} + +func TestTheFailureOfTheUnitTheModuleTakesOverIsCleared(t *testing.T) { + f := newSystem(t) + f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n") + f.write(ExportsFile, exportsAnswer(t, library)) + f.reachable["192.0.2.10:2049"] = true + // fstab's generator made srv-library.mount for the same mount point, and it timed out at boot. + f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"} + + states := failedLook(t, f) + + if !f.called("systemctl reset-failed srv-library.mount") { + t.Fatalf("the generator's failure is cleared once the module's unit is written: %q", f.calls) + } + if s := states["library"]; s.State != StateArmed { + t.Errorf("the share is judged on its own mounts, not on the generator's: %+v", s) + } +} + +func TestAMountThatFailedBecauseItsSourceWasAwayIsCleared(t *testing.T) { + f := newSystem(t) + f.write(ConfigFile, "shares=library=/srv/library\nsources=games=smb://192.0.2.76/Share@/mnt/games\n"+ + "server=homeserver\nat=192.0.2.10\nnode=workstation\naccount=operator\n"+ + "secret=/var/lib/mesh/mounts/smb-credentials.secret\nstate=/var/lib/mesh/mounts\n") + f.write(ExportsFile, exportsAnswer(t, library)) + f.write("/var/lib/mesh/mounts/smb-credentials.secret", "[games]\nusername=root\npassword="+testPassword+"\n") + a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)) + a.Look(context.Background()) // writes and arms both pairs; neither source answers + f.calls = nil + + // Someone opened both mount points: the gaming device is off and the server is away, so both mounts failed. + f.units["mnt-games.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"} + f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"} + cleared := a.clearFailed(context.Background()) + + for _, unit := range []string{"mnt-games.mount", "srv-library.mount"} { + if !f.called("systemctl reset-failed " + unit) { + t.Errorf("%s failed because its source was away and is cleared: %q", unit, f.calls) + } + } + if len(cleared) != 2 || !strings.Contains(strings.Join(cleared, "; "), "games was not there to mount") { + t.Errorf("each clearing is said: %q", cleared) + } + states := failedLook(t, f) + if states["games"].State != StateAbsent || states["library"].State != StateUnreachable { + t.Errorf("the module still says each as absent or unreachable: %+v", states) + } +} + +func TestAMountThatFailedWhileItsSourceAnswersStaysFailed(t *testing.T) { + f := newSystem(t) + f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n") + f.write(ExportsFile, exportsAnswer(t, library)) + f.reachable["192.0.2.10:2049"] = true + a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)) + a.Look(context.Background()) + f.calls = nil + + f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "exit-code"} + if cleared := a.clearFailed(context.Background()); len(cleared) != 0 || f.called("systemctl reset-failed") { + t.Fatalf("a mount refused by a server that answers is a fault and is left failed: %q %q", cleared, f.calls) + } + if s := failedLook(t, f)["library"]; s.State != StateFailed { + t.Errorf("it is said failed: %+v", s) + } +} + +func TestOnlyTheActingProcessClears(t *testing.T) { + f := newSystem(t) + f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n") + f.write(ExportsFile, exportsAnswer(t, library)) + f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"} + a := f.applier(time.Now()) + a.Act = false + a.Look(context.Background()) + if f.called("systemctl reset-failed") { + t.Errorf("the judging process changes nothing: %q", f.calls) + } +} diff --git a/modules/mounts/cmd/mounts/write_test.go b/modules/mounts/cmd/mounts/write_test.go index bf06d3d4..7d6768c1 100644 --- a/modules/mounts/cmd/mounts/write_test.go +++ b/modules/mounts/cmd/mounts/write_test.go @@ -77,6 +77,11 @@ func (f *fakeSystem) run(_ context.Context, name string, args ...string) (string case "enable": u := args[len(args)-1] f.units[u] = map[string]string{"ActiveState": "active", "UnitFileState": "enabled"} + case "reset-failed": + u := args[len(args)-1] + if f.units[u]["ActiveState"] == "failed" { + f.units[u]["ActiveState"], f.units[u]["Result"] = "inactive", "success" + } } return "", nil }