mesh-catalog/modules/mounts/cmd/mounts/failed_test.go

package main

import (
	"context"
	"strings"
	"testing"
	"time"
)

// A mount the module took over, or one whose source was away when it was touched, is never left as a
// failed unit: the node-engine would say it as a failed unit of the machine's own, with a notification each
// time (hq ADR 0263 rules 4 and 6). A mount that failed while its source answers stays failed: a fault.

func failedLook(t *testing.T, f *fakeSystem) map[string]ShareStatus {
	t.Helper()
	st := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)).Look(context.Background())
	out := map[string]ShareStatus{}
	for _, s := range st.Shares {
		out[s.Name] = s
	}
	return out
}

func TestTheFailureOfTheUnitTheModuleTakesOverIsCleared(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
	f.write(ExportsFile, exportsAnswer(t, library))
	f.reachable["192.0.2.10:2049"] = true
	// fstab's generator made srv-library.mount for the same mount point, and it timed out at boot.
	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}

	states := failedLook(t, f)

	if !f.called("systemctl reset-failed srv-library.mount") {
		t.Fatalf("the generator's failure is cleared once the module's unit is written: %q", f.calls)
	}
	if s := states["library"]; s.State != StateArmed {
		t.Errorf("the share is judged on its own mounts, not on the generator's: %+v", s)
	}
}

func TestAMountThatFailedBecauseItsSourceWasAwayIsCleared(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, "shares=library=/srv/library\nsources=games=smb://192.0.2.76/Share@/mnt/games\n"+
		"server=homeserver\nat=192.0.2.10\nnode=workstation\naccount=operator\n"+
		"secret=/var/lib/mesh/mounts/smb-credentials.secret\nstate=/var/lib/mesh/mounts\n")
	f.write(ExportsFile, exportsAnswer(t, library))
	f.write("/var/lib/mesh/mounts/smb-credentials.secret", "[games]\nusername=root\npassword="+testPassword+"\n")
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background()) // writes and arms both pairs; neither source answers
	f.calls = nil

	// Someone opened both mount points: the gaming device is off and the server is away, so both mounts failed.
	f.units["mnt-games.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
	cleared := a.clearFailed(context.Background())

	for _, unit := range []string{"mnt-games.mount", "srv-library.mount"} {
		if !f.called("systemctl reset-failed " + unit) {
			t.Errorf("%s failed because its source was away and is cleared: %q", unit, f.calls)
		}
	}
	if len(cleared) != 2 || !strings.Contains(strings.Join(cleared, "; "), "games was not there to mount") {
		t.Errorf("each clearing is said: %q", cleared)
	}
	states := failedLook(t, f)
	if states["games"].State != StateAbsent || states["library"].State != StateUnreachable {
		t.Errorf("the module still says each as absent or unreachable: %+v", states)
	}
}

func TestAMountThatFailedWhileItsSourceAnswersStaysFailed(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
	f.write(ExportsFile, exportsAnswer(t, library))
	f.reachable["192.0.2.10:2049"] = true
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background())
	f.calls = nil

	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "exit-code"}
	if cleared := a.clearFailed(context.Background()); len(cleared) != 0 || f.called("systemctl reset-failed") {
		t.Fatalf("a mount refused by a server that answers is a fault and is left failed: %q %q", cleared, f.calls)
	}
	if s := failedLook(t, f)["library"]; s.State != StateFailed {
		t.Errorf("it is said failed: %+v", s)
	}
}

func TestOnlyTheActingProcessClears(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, "shares=library=/srv/library\nserver=homeserver\nat=192.0.2.10\nnode=workstation\n")
	f.write(ExportsFile, exportsAnswer(t, library))
	f.units["srv-library.mount"] = map[string]string{"ActiveState": "failed", "Result": "timeout"}
	a := f.applier(time.Now())
	a.Act = false
	a.Look(context.Background())
	if f.called("systemctl reset-failed") {
		t.Errorf("the judging process changes nothing: %q", f.calls)
	}
}

// waitFor polls until cond holds or the deadline passes.
func waitFor(t *testing.T, within time.Duration, cond func() bool) bool {
	t.Helper()
	deadline := time.Now().Add(within)
	for time.Now().Before(deadline) {
		if cond() {
			return true
		}
		time.Sleep(5 * time.Millisecond)
	}
	return cond()
}

func TestTheActingProcessClearsBetweenItsLooks(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, libraryOnly)
	f.write(ExportsFile, exportsAnswer(t, library))
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Judge = false
	// The next look is an hour away: only the loop between looks can clear what fails now.
	a.LookInterval, a.ClearInterval = time.Hour, 10*time.Millisecond
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { RunApply(ctx, a); close(done) }()
	defer func() { cancel(); <-done }()

	if !waitFor(t, 2*time.Second, func() bool { return f.called("systemctl enable --now srv-library.automount") }) {
		t.Fatal("the first look did not arm the pair")
	}
	// The server is away and someone opened the mount point: the mount failed after the look.
	f.setUnit("srv-library.mount", map[string]string{"ActiveState": "failed", "Result": "timeout"})
	if !waitFor(t, 2*time.Second, func() bool { return f.called("systemctl reset-failed srv-library.mount") }) {
		t.Fatal("a mount that failed between looks, its server away, is cleared before the next look")
	}
}

func TestALookClearsAMountOfItsOwnThatFailedWhileItsSourceWasAway(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, libraryOnly)
	f.write(ExportsFile, exportsAnswer(t, library))
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background())
	f.setUnit("srv-library.mount", map[string]string{"ActiveState": "failed", "Result": "timeout"})
	f.calls = nil

	// A second look, with nothing to write: the pair is the module's own already, so this is no takeover.
	a.Look(context.Background())
	if !f.called("systemctl reset-failed srv-library.mount") {
		t.Errorf("the look itself clears it: %q", f.calls)
	}
}

func TestAnAbsentDiskIsClearedAndAPresentOneIsNot(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, "sources=usb=disk:1234-ABCD@/mnt/usb\nnode=workstation\n")
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background())

	f.setUnit("mnt-usb.mount", map[string]string{"ActiveState": "failed", "Result": "timeout"})
	f.calls = nil
	if a.clearFailed(context.Background()); !f.called("systemctl reset-failed mnt-usb.mount") {
		t.Errorf("the disk is not plugged in: its failed mount is cleared: %q", f.calls)
	}

	// Plugged in, and the mount still failed: a fault, left as it is.
	f.write("/dev/disk/by-uuid/1234-ABCD", "")
	f.setUnit("mnt-usb.mount", map[string]string{"ActiveState": "failed", "Result": "exit-code"})
	f.calls = nil
	if a.clearFailed(context.Background()); f.called("systemctl reset-failed") {
		t.Errorf("a disk that is there and failed to mount is left failed: %q", f.calls)
	}
}

func TestARewrittenPairOfItsOwnKeepsItsFailure(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, libraryOnly)
	f.write(ExportsFile, exportsAnswer(t, library))
	f.reachable["192.0.2.10:2049"] = true
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background())

	// The server moved the library and refuses the mount while it answers: the module's own failure.
	f.setUnit("srv-library.mount", map[string]string{"ActiveState": "failed", "Result": "exit-code"})
	moved := library
	moved.Path = "/tank/library"
	f.write(ExportsFile, exportsAnswer(t, moved))
	f.calls = nil
	a.Look(context.Background())
	if !strings.Contains(f.read(UnitDir+"/srv-library.mount"), "What=192.0.2.10:/tank/library") {
		t.Fatal("the pair was not rewritten")
	}
	if f.called("systemctl reset-failed") {
		t.Errorf("only a unit the module takes over has its failure cleared, not one of its own it rewrites: %q", f.calls)
	}
}

func TestAnAutomountThatGaveUpIsClearedAndArmedAgain(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, libraryOnly)
	f.write(ExportsFile, exportsAnswer(t, library))
	a := f.applier(time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC))
	a.Look(context.Background())

	f.setUnit("srv-library.automount", map[string]string{"ActiveState": "failed", "Result": "mount-start-limit-hit", "UnitFileState": "enabled"})
	f.calls = nil
	cleared := a.clearFailed(context.Background())
	if !f.called("systemctl reset-failed srv-library.automount") || !f.called("systemctl start srv-library.automount") {
		t.Fatalf("the automount is cleared and started: %q", f.calls)
	}
	if !strings.Contains(strings.Join(cleared, ";"), "mount-start-limit-hit") {
		t.Errorf("said with its result: %q", cleared)
	}

	// With the server answering, an automount that gave up is a fault: left as it is.
	f.reachable["192.0.2.10:2049"] = true
	f.setUnit("srv-library.automount", map[string]string{"ActiveState": "failed", "Result": "mount-start-limit-hit", "UnitFileState": "enabled"})
	f.calls = nil
	if a.clearFailed(context.Background()); f.called("systemctl reset-failed") || f.called("systemctl start") {
		t.Errorf("left failed while the server answers: %q", f.calls)
	}
}

func TestATransientUnitOfTheSameNameIsNotTheModules(t *testing.T) {
	f := newSystem(t)
	f.write(ConfigFile, libraryOnly)
	f.write(ExportsFile, exportsAnswer(t, library))
	f.write(TransientDir+"/srv-library.mount", "[Mount]\nWhat=elsewhere\n")
	states := failedLook(t, f)
	if s := states["library"]; s.State != StateFailed || !strings.Contains(s.Why, "transient") {
		t.Errorf("a share whose unit someone made at run time is refused: %+v", s)
	}
	if f.called("systemctl enable --now srv-library.automount") {
		t.Error("and not armed")
	}
}