mesh-host/internal/apply/kept_test.go

//go:build linux

package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/sys/unix"

	"github.com/novox/mesh-host/internal/store"
)

// The engine's half of ADR 0325's checks (novox/hq ADR 0325, "How it is checked", issue 567): a shared folder kept
// through access entries, on a filesystem of the test's own. Nothing here needs root: a file's owner may set its
// ACL, and the identities are numbers no account here has.

const (
	radarr = 61001 // a container's identity, read-write
	plex   = 61002 // a container's identity, read
	runner = 61003 // the tool runner's account
)

// aclFolder is a folder of the test's own on a filesystem that carries ACLs, or the test is skipped saying why.
func aclFolder(t *testing.T) string {
	t.Helper()
	dir := t.TempDir()
	fd, err := unix.Open(dir, unix.O_RDONLY|unix.O_DIRECTORY, 0)
	if err != nil {
		t.Fatal(err)
	}
	defer unix.Close(fd)
	probe := keptACL(minimalACL(0o755), map[uint32]uint16{radarr: 5})
	if err := setACL(fd, aclAccessXattr, probe); errors.Is(err, ErrNoACL) {
		t.Skipf("the test's filesystem (%s) carries no POSIX ACLs (ENOTSUP), and every check here reads them", dir)
	} else if err != nil {
		t.Fatal(err)
	}
	if err := unix.Fremovexattr(fd, aclAccessXattr); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(dir, 0o755); err != nil {
		t.Fatal(err)
	}
	return dir
}

// keeping has the test keep its own record of kept folders, and judge mounts as the test says.
func keeping(t *testing.T, points []string, targets []string) {
	t.Helper()
	was, wasPoints, wasTargets := KeptFolders, MountPoints, MountTargets
	KeptFolders = NewKeptMemory()
	forgetMountHistory()
	MountPoints = func() []string { return append([]string{"/"}, points...) }
	MountTargets = func(context.Context) []string { return targets }
	t.Cleanup(func() {
		KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets
		forgetMountHistory()
	})
}

// accessJSON is an access of module to path with users, each "identity:uid:use:from".
func accessJSON(id, path, mode string, taken bool, users ...string) string {
	var us []string
	for _, u := range users {
		f := strings.Split(u, ":")
		us = append(us, fmt.Sprintf(`{"identity":%q,"uid":%s,"use":%q,"from":%q}`, f[0], f[1], f[2], f[3]))
	}
	return fmt.Sprintf(`{"id":%q,"type":"access","path":%q,"mode":%q,"taken":%v,"users":[%s]}`, id, path, mode,
		taken, strings.Join(us, ","))
}

// readACLs is a path's access ACL and default ACL (nil when it has none), and its stat.
func readACLs(t *testing.T, path string) (acl, acl, unix.Stat_t) {
	t.Helper()
	fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NOFOLLOW, 0)
	if err != nil {
		t.Fatal(err)
	}
	defer unix.Close(fd)
	var st unix.Stat_t
	if err := unix.Fstat(fd, &st); err != nil {
		t.Fatal(err)
	}
	access, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777)
	if err != nil {
		t.Fatal(err)
	}
	def, err := getACL(fd, aclDefaultXattr, uint32(st.Mode)&0o7777)
	if err != nil {
		t.Fatal(err)
	}
	return access, def, st
}

// effectiveOn is what uid may do on path, by entry and mask together.
func effectiveOn(t *testing.T, path string, uid uint32) uint16 {
	t.Helper()
	a, _, st := readACLs(t, path)
	return effective(a, uid, st.Uid, uint32(st.Mode)&0o7777)
}

func applyAccesses(t *testing.T, known store.State, resources ...string) (Report, store.State, error) {
	t.Helper()
	d := declare(t, strings.Join(resources, ","))
	return Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, noServices, nil, nil)
}

// Row 5: entries and default entries per identity and use; owner, owning group, owner's and others' bits unchanged;
// the owning group's entry equals its former group bits; the mask equals what the entries need; an entry the mesh
// did not add is untouched.
func TestAKeptFolderCarriesAnEntryPerUserAndNothingElseOfItChanges(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "movies")
	if err := os.Mkdir(lib, 0o750); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(lib, 0o750); err != nil {
		t.Fatal(err)
	}
	// The operator's own entry, there before: never the mesh's to change.
	fd, _ := unix.Open(lib, unix.O_RDONLY|unix.O_DIRECTORY, 0)
	operator := keptACL(minimalACL(0o750), map[uint32]uint16{60099: permRead})
	operator[len(operator)-1].Perm = permRead | permExec // its mask: r-x, as the operator set it
	if err := setACL(fd, aclAccessXattr, operator); err != nil {
		t.Fatal(err)
	}
	unix.Close(fd)
	_, _, before := readACLs(t, lib)

	report, _, err := applyAccesses(t, store.State{},
		accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container"),
		accessJSON("plex.movies", lib, "read", false, "mesh-plex:61002:read:container"))
	if err != nil {
		t.Fatalf("an empty folder was not kept: %v", err)
	}
	access, def, after := readACLs(t, lib)
	if bits, _ := access.perm(aclUser, radarr); bits != 7 {
		t.Errorf("the read-write identity's entry is %s, and it needs rwx", permWords(bits))
	}
	if bits, _ := access.perm(aclUser, plex); bits != 5 {
		t.Errorf("the read identity's entry is %s, and it needs r-x", permWords(bits))
	}
	for _, uid := range []uint32{radarr, plex} {
		if _, ok := def.perm(aclUser, uid); !ok {
			t.Errorf("uid %d has no default entry: what is made under the folder would not be its", uid)
		}
	}
	if bits, _ := access.perm(aclUser, 60099); bits != permRead {
		t.Errorf("the operator's entry was changed to %s", permWords(bits))
	}
	if g, _ := access.perm(aclGroupObj, 0); g != 5 {
		t.Errorf("the owning group's entry is %s, and its group bits gave r-x", permWords(g))
	}
	if m, _ := access.perm(aclMask, 0); m != 7 {
		t.Errorf("the mask is %s, and the entries need rwx", permWords(m))
	}
	if after.Uid != before.Uid || after.Gid != before.Gid {
		t.Error("the folder's owner or owning group changed")
	}
	if after.Mode&0o707 != before.Mode&0o707 {
		t.Errorf("the owner's or others' bits changed: %o to %o", before.Mode&0o777, after.Mode&0o777)
	}
	if after.Mode&0o070 != 0o070 {
		t.Errorf("the group bits show %o, and they show the mask the entries need (rwx)", (after.Mode>>3)&7)
	}
	rec, _ := keptRecord(lib)
	if !rec.Kept || rec.How != store.KeptEmpty {
		t.Errorf("the folder was not recorded kept as one that held nothing: %+v", rec)
	}
	for _, o := range report.Outcomes {
		if o.ID == "radarr.movies" && o.Action == "waiting" {
			t.Errorf("an empty folder waited for a warrant: %s", o.Detail)
		}
	}
}

// Row 4: the runner's entry is the widest use among the modules whose tools use the folder.
func TestTheToolRunnersEntryIsTheWidestUseAnyModuleGivesIt(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "books")
	if err := os.Mkdir(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	_, _, err := applyAccesses(t, store.State{},
		accessJSON("reader.books", lib, "read", false, "nox:61003:read:tool-runner"),
		accessJSON("importer.books", lib, "read-write", false, "nox:61003:read-write:tool-runner"))
	if err != nil {
		t.Fatal(err)
	}
	if got := effectiveOn(t, lib, runner); got != 7 {
		t.Fatalf("the tool runner may %s, and one module whose tools use it declared read-write", permWords(got))
	}
}

// Row 6: a file made at 0666 by one identity is writable by another read-write identity at once; one made at 0644
// is writable by it after the watch's repair, and before it the judgement says so; with watches exhausted, the
// sweep repairs it; the judgement reads the effective right (entry and mask).
func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "downloads")
	if err := os.Mkdir(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	users := []string{"mesh-radarr:61001:read-write:container", "mesh-sab:61004:read-write:container"}
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.downloads", lib, "read-write", false,
		users...)); err != nil {
		t.Fatal(err)
	}
	rec, _ := keptRecord(lib)

	usual := filepath.Join(lib, "usual")
	if err := os.WriteFile(usual, nil, 0o666); err != nil {
		t.Fatal(err)
	}
	if got := effectiveOn(t, usual, radarr); got&6 != 6 {
		t.Fatalf("a file made at 0666 gives another read-write identity %s at once", permWords(got))
	}

	w, err := NewWatcher()
	if err != nil {
		t.Skipf("no inotify here: %v", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	ended := make(chan struct{})
	go func() { w.Run(ctx); close(ended) }()
	defer func() { cancel(); <-ended }()
	w.Set(map[string][]store.KeptUser{lib: rec.Want})

	narrow := filepath.Join(lib, "narrow")
	f, err := os.OpenFile(narrow, os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	waitFor(t, "the watch widened the mask of a file made at 0644", func() bool {
		return effectiveOn(t, narrow, radarr)&6 == 6
	})

	// Before a repair the judgement says so: a mask narrowed by hand where nothing watches.
	w.Set(nil)
	if err := os.Chmod(usual, 0o644); err != nil {
		t.Fatal(err)
	}
	c, err := JudgeKept(lib, rec.Want, ActLimits)
	if err != nil {
		t.Fatal(err)
	}
	if c.Narrowed == 0 {
		t.Fatal("a file whose mask gives a read-write user only read is not said by the judgement")
	}

	// Watches run out: the record says so, and the sweep puts it back.
	was := MaxWatchedPerFolder
	MaxWatchedPerFolder = 0
	defer func() { MaxWatchedPerFolder = was }()
	w.Set(map[string][]store.KeptUser{lib: rec.Want})
	if r, _ := keptRecord(lib); !r.OutOfWatches {
		t.Error("watches that ran out are not recorded")
	}
	look := LookKept(lib, rec.Want, SweepLimits)
	if look.Repaired == 0 {
		t.Errorf("the sweep put nothing back: %+v", look)
	}
	if got := effectiveOn(t, usual, radarr); got&6 != 6 {
		t.Fatalf("after the sweep a read-write identity may %s", permWords(got))
	}
	if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 {
		t.Fatalf("after the sweep %d entries still give a user less than its right", c.Narrowed)
	}
}

func waitFor(t *testing.T, what string, ok func() bool) {
	t.Helper()
	deadline := time.Now().Add(3 * time.Second)
	for !ok() {
		if time.Now().After(deadline) {
			t.Fatalf("not within 3s: %s", what)
		}
		time.Sleep(20 * time.Millisecond)
	}
}

// Row 8, the engine's part: a folder that holds data waits, and nothing is changed without a warrant; the act adds
// the entries under the folder, follows no link, enters no other filesystem; a dropped access removes exactly its
// own entries.
func TestAFolderThatHoldsDataIsTakenOnlyOnAWarrantAndLetGoEntryByEntry(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "music")
	outside := filepath.Join(dir, "outside")
	for _, d := range []string{lib, filepath.Join(lib, "album"), outside} {
		if err := os.Mkdir(d, 0o755); err != nil {
			t.Fatal(err)
		}
	}
	song := filepath.Join(lib, "album", "song")
	if err := os.WriteFile(song, []byte("data"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.Symlink(outside, filepath.Join(lib, "link")); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("lidarr.music", lib, "read-write", false, "mesh-lidarr:61001:read-write:container",
		"mesh-plex:61002:read:container")
	report, _, err := applyAccesses(t, store.State{}, a)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "lidarr.music"); o.Action != "waiting" || !strings.Contains(o.Detail, KeepWords("music")) {
		t.Fatalf("a folder that holds data did not wait for the operator: %+v", o)
	}
	if has := effectiveOn(t, lib, radarr); has != 0 {
		t.Fatal("the folder was given an entry without a warrant")
	}
	if strings.Contains(KeepWords("music"), "/") || strings.Contains(DiskWords("music"), "/") {
		t.Fatal("a wait's words name a path")
	}

	d := declare(t, a)
	done, said, err := TakeFolder(context.Background(), d, "lidarr.music", lib, "warrant-7", "the operator", noServices)
	if err != nil {
		t.Fatalf("the take act was refused: %v", err)
	}
	<-done
	if !strings.Contains(said, "warrant-7") {
		t.Errorf("the act does not say its warrant: %s", said)
	}
	for _, p := range []string{lib, filepath.Join(lib, "album"), song} {
		if got := effectiveOn(t, p, radarr); got&6 != 6 {
			t.Errorf("after the act the read-write identity may %s", permWords(got))
		}
	}
	if has := effectiveOn(t, outside, radarr); has != 0 {
		t.Error("the act followed a link out of the folder")
	}
	rec, _ := keptRecord(lib)
	if !rec.Kept || rec.How != store.KeptByWarrant || rec.Warrant != "warrant-7" || rec.Act == nil ||
		rec.Act.State != "done" || rec.Act.Links != 1 {
		t.Fatalf("the act is not recorded as done under its warrant, with the link passed by: %+v %+v", rec, rec.Act)
	}

	// Kept now: the next apply keeps it without asking, and plex dropping it has exactly plex's entries removed.
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false,
		"mesh-lidarr:61001:read-write:container")); err != nil {
		t.Fatal(err)
	}
	beforeMask, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }()
	ReconcileKept(context.Background())
	for _, p := range []string{lib, song} {
		a, def, _ := readACLs(t, p)
		if _, still := a.perm(aclUser, plex); still {
			t.Errorf("a dropped user's entry stayed")
		}
		if _, still := def.perm(aclUser, plex); still && def != nil {
			t.Errorf("a dropped user's default entry stayed")
		}
		if _, kept := a.perm(aclUser, radarr); !kept {
			t.Errorf("the user that stayed lost its entry")
		}
	}
	if m, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }(); m != beforeMask {
		t.Errorf("the mask moved on a drop: %s to %s", permWords(beforeMask), permWords(m))
	}
}

// Row 8: the act enters no other filesystem — a folder whose device is not the kept folder's is passed by.
func TestAWalkEntersNoOtherFilesystem(t *testing.T) {
	dir := aclFolder(t)
	mount := filepath.Join(dir, "elsewhere")
	if err := os.Mkdir(mount, 0o755); err != nil {
		t.Fatal(err)
	}
	inside := filepath.Join(mount, "file")
	if err := os.WriteFile(inside, nil, 0o644); err != nil {
		t.Fatal(err)
	}
	// A stand-in for a device the test cannot mount: the folder named elsewhere reads as another filesystem.
	stats := walkStats{
		at: func(dirfd int, name string, st *unix.Stat_t) error {
			err := unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW)
			if name == "elsewhere" {
				st.Dev++
			}
			return err
		},
		fd: unix.Fstat,
	}
	w, err := walkKept(dir, ActLimits, stats, nil, keepVisit(map[uint32]bool{radarr: true}))
	if err != nil {
		t.Fatal(err)
	}
	if w.counts.Mounts != 1 {
		t.Errorf("the walk counted %d folders on another filesystem, and there is one", w.counts.Mounts)
	}
	if got := effectiveOn(t, inside, radarr); got != 0 {
		t.Error("the walk entered another filesystem")
	}
}

// Row 9: a missing folder under a mounted dataset is made as its parent; with the dataset not mounted (its empty
// mount point on the root filesystem) nothing is made and the access waits.
func TestAMissingFolderIsMadeWhereItsDataLivesAndWaitsWhereItsDiskIsNot(t *testing.T) {
	dir := aclFolder(t)
	dataset := filepath.Join(dir, "tank")
	if err := os.Mkdir(dataset, 0o750); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(dataset, 0o750); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{dataset}, []string{dataset})
	movies := filepath.Join(dataset, "media", "movies")
	report, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", movies, "read-write", false,
		"mesh-radarr:61001:read-write:container"))
	if err != nil {
		t.Fatalf("a missing folder under a mounted dataset was not made: %v", err)
	}
	info, err := os.Stat(movies)
	if err != nil {
		t.Fatalf("the folder was not made: %v", err)
	}
	if info.Mode().Perm()&0o707 != 0o700 {
		t.Errorf("the folder was made at %o, and its parent is 0750", info.Mode().Perm())
	}
	if got := effectiveOn(t, movies, radarr); got != 7 {
		t.Errorf("the folder made is not kept: the identity may %s", permWords(got))
	}
	if rec, _ := keptRecord(movies); rec.How != store.KeptMade || rec.Mount != dataset {
		t.Errorf("not recorded as made on its dataset: %+v", rec)
	}
	_ = report

	// The dataset not mounted: its mount point an empty folder on the filesystem below.
	MountPoints = func() []string { return []string{"/"} }
	books := filepath.Join(dataset, "books")
	report, _, err = applyAccesses(t, store.State{}, accessJSON("readarr.books", books, "read-write", false,
		"mesh-readarr:61001:read-write:container"))
	if err != nil {
		t.Fatalf("a missing disk failed the apply: %v", err)
	}
	if o := outcomeOf(report, "readarr.books"); o.Action != "waiting" || !strings.Contains(o.Detail, DiskWords("books")) {
		t.Fatalf("a folder whose disk is not there did not wait for it: %+v", o)
	}
	if _, err := os.Stat(books); !errors.Is(err, os.ErrNotExist) {
		t.Fatal("a folder was made where its disk is not")
	}
	if v := AccessVerdict(context.Background(), books, false, true); v.Kind != VerdictWaitDisk {
		t.Errorf("the look judges it %s, not waiting for its disk", v.Kind)
	}
	// And the folder made before, its dataset gone: its disk is not there, whatever is at its path now.
	if v := AccessVerdict(context.Background(), movies, false, true); v.Kind != VerdictWaitDisk {
		t.Errorf("a kept folder whose dataset is not mounted is judged %s", v.Kind)
	}
}

// Row 10: a narrowed entry or mask is reported and put back; successive looks cover a folder whole; a look over
// its bound says how far it got.
func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "shows")
	if err := os.Mkdir(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	rec, _ := keptRecord(lib)
	for i := 0; i < 25; i++ {
		p := filepath.Join(lib, fmt.Sprintf("episode-%02d", i))
		if err := os.WriteFile(p, nil, 0o666); err != nil {
			t.Fatal(err)
		}
		if err := os.Chmod(p, 0o644); err != nil { // an app's chmod narrows the mask
			t.Fatal(err)
		}
	}
	// The folder itself narrowed too: put back on the next apply, said by the look until then.
	if err := os.Chmod(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	if short, _ := judgeTop(lib, rec.Want); len(short) == 0 {
		t.Fatal("a folder whose mask an app narrowed is not said short")
	}
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	if short, _ := judgeTop(lib, rec.Want); len(short) != 0 {
		t.Fatalf("the apply did not put the folder back: %v", short)
	}

	limits := WalkLimits{Entries: 10, Time: time.Minute}
	looks, repaired := 0, 0
	for {
		looks++
		look := LookKept(lib, rec.Want, limits)
		repaired += look.Repaired
		if look.Failed != "" {
			t.Fatal(look.Failed)
		}
		if !look.Sweep.Bound || look.Sweep.Cursor == nil {
			break
		}
		if len(look.Sweep.Cursor) == 0 || look.Sweep.Read == 0 {
			t.Fatalf("a look over its bound does not say how far it got: %+v", look.Sweep)
		}
		if looks > 10 {
			t.Fatal("the sweep never covered the folder")
		}
	}
	if looks < 3 {
		t.Errorf("26 entries at 10 a look were covered in %d looks", looks)
	}
	if repaired != 25 {
		t.Errorf("the sweep put back %d of 25 narrowed entries", repaired)
	}
	if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 {
		t.Errorf("after the looks %d entries still give the user less than its right", c.Narrowed)
	}
	if r, _ := keptRecord(lib); r.Sweep.Covered.IsZero() {
		t.Error("a sweep that covered the folder is not recorded as having done so")
	}
}

// Row 12: a root-owned folder on its placed filesystem, and an absent one under a root-owned parent on it, are
// unhealthy, nothing is made, and no ask is raised. Read from the machine's own root-owned folders, never changed.
func TestRootsDataOnItsPlacedFilesystemIsNeverKept(t *testing.T) {
	keeping(t, nil, nil)
	// A root-owned folder the placement guard lets an access name: below a protected root, never one.
	root := ""
	for _, c := range []string{"/srv/http", "/srv/ftp", "/var/empty", "/var/opt"} {
		if info, err := os.Lstat(c); err == nil && info.IsDir() {
			if uid, _, _ := statOwner(info); uid == 0 {
				root = c
				break
			}
		}
	}
	if root == "" {
		t.Skip("this machine has no root-owned folder below /srv or /var to read")
	}
	absent := filepath.Join(root, "mesh-test-never-made")
	for _, path := range []string{root, absent} {
		report, _, err := applyAccesses(t, store.State{}, accessJSON("m.data", path, "read-write", false,
			"mesh-m:61001:read-write:container"))
		if err == nil || !strings.Contains(err.Error(), "root's data is never kept") {
			t.Errorf("root's data at %s was not refused for what it is: %v", path, err)
		}
		if o := outcomeOf(report, "m.data"); o.Action == "waiting" {
			t.Errorf("root's data waits, and is never asked about: %+v", o)
		}
		if v := AccessVerdict(context.Background(), path, false, true); v.Kind != VerdictRoot {
			t.Errorf("the look judges root's data at %s %s", path, v.Kind)
		}
	}
	if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) {
		t.Fatal("a folder was made under root's data")
	}
	if _, has := keptRecord(root); has {
		t.Error("root's data was recorded as a kept folder")
	}
}

// The codec reads back what it writes, in the kernel's order.
func TestAnACLReadsBackAsWritten(t *testing.T) {
	a := keptACL(minimalACL(0o640), map[uint32]uint16{radarr: 6, plex: 4})
	b, err := decodeACL(a.encode())
	if err != nil {
		t.Fatal(err)
	}
	if !sameACL(a, b) || b[0].Tag != aclUserObj || b[len(b)-1].Tag != aclOther {
		t.Fatalf("read back %+v", b)
	}
	if err := b.check(); err != nil {
		t.Fatal(err)
	}
	if _, err := decodeACL([]byte{1, 0, 0, 0}); err == nil {
		t.Error("a version 1 attribute was read")
	}
	if m, _ := b.perm(aclMask, 0); m != 6 {
		t.Errorf("mask %s, and the entries need rw- and the group had r--", permWords(m))
	}
}

func forgetMountHistory() {
	mountHistory.Lock()
	mountHistory.seen = map[string]bool{}
	mountHistory.Unlock()
}

// Review H1: a walk is judged as an apply is, first. With the dataset not mounted its mount point is what is at the
// path, and neither an add nor a drop walk touches it; the record stays for the walk that can reach the data.
func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) {
	dir := aclFolder(t)
	dataset := filepath.Join(dir, "tank")
	if err := os.Mkdir(dataset, 0o755); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{dataset}, nil)
	lib := filepath.Join(dataset, "movies")
	both := []string{"mesh-radarr:61001:read-write:container", "mesh-plex:61002:read:container"}
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
		both...)); err != nil {
		t.Fatal(err)
	}
	if rec, _ := keptRecord(lib); rec.Mount != dataset || !sameUsers(rec.Users, rec.Want) {
		t.Fatalf("not kept on its dataset: %+v", rec)
	}
	// plex drops it, and the dataset is unmounted before the walk.
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
		both[0])); err != nil {
		t.Fatal(err)
	}
	MountPoints = func() []string { return []string{"/"} }
	ReconcileKept(context.Background())
	rec, had := keptRecord(lib)
	if !had || len(rec.Users) != 2 {
		t.Fatalf("a drop that could not reach the data forgot whose entries are on it: %+v", rec)
	}
	if rec.Act == nil || rec.Act.State != "waiting" {
		t.Errorf("the walk not run is not said: %+v", rec.Act)
	}
	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); !still {
		t.Fatal("a walk ran where the disk is not there")
	}
	// The disk back: the drop runs.
	MountPoints = func() []string { return []string{"/", dataset} }
	ReconcileKept(context.Background())
	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); still {
		t.Fatal("the drop did not run once the disk was there")
	}
}

// Review H2: a folder kept while it was empty on the filesystem below a mount nobody named is judged afresh when
// the disk mounts over it: holding data, it waits for the operator's word, and nothing is added to it.
func TestADiskMountedOverAFolderKeptEmptyIsNotKeptWithoutAWarrant(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	pool := filepath.Join(dir, "pool")
	lib := filepath.Join(pool, "media")
	if err := os.MkdirAll(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	if rec, _ := keptRecord(lib); !rec.Kept || rec.How != store.KeptEmpty || rec.Mount != "/" {
		t.Fatalf("not kept as empty on /: %+v", rec)
	}
	// The pool is imported: a library is at the path now, on its own filesystem.
	film := filepath.Join(lib, "film")
	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
		t.Fatal(err)
	}
	// The new disk's data carries no entries of the mesh's: the test's stand-in made it under the folder kept
	// below, so what it inherited there is taken off.
	if err := unix.Removexattr(film, aclAccessXattr); err != nil {
		t.Fatal(err)
	}
	MountPoints = func() []string { return []string{"/", pool} }
	if v := AccessVerdict(context.Background(), lib, false, true); v.Kind != VerdictWaitKeep {
		t.Fatalf("the look judges it %s, and the data on the new disk was never taken", v.Kind)
	}
	report, _, err := applyAccesses(t, store.State{}, a)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("a folder whose disk mounted over it was kept without a warrant: %+v", o)
	}
	if rec, _ := keptRecord(lib); rec.Kept || rec.Mount != pool {
		t.Errorf("the record still says kept, or not on the new disk: %+v", rec)
	}
	if got := effectiveOn(t, film, radarr); got != 0 {
		t.Error("an entry was added to the data on the new disk")
	}
}

// Review M3: widening a mask widens nobody else — a file made at 0644 under a default ACL carries the owning group's
// rwx under a mask of r--, and after the mesh widens the mask the owning group still reads only.
func TestWideningTheMaskGivesTheOwningGroupNothingMore(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "shows")
	if err := os.Mkdir(lib, 0o775); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(lib, 0o775); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	ep := filepath.Join(lib, "episode")
	f, err := os.OpenFile(ep, os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	groupRight := func() uint16 {
		acl, _, _ := readACLs(t, ep)
		g, _ := acl.perm(aclGroupObj, 0)
		m, _ := acl.perm(aclMask, 0)
		return g & m
	}
	before := groupRight()
	rec, _ := keptRecord(lib)
	LookKept(lib, rec.Want, SweepLimits)
	if got := effectiveOn(t, ep, radarr); got&6 != 6 {
		t.Fatalf("the sweep did not widen the mask: %s", permWords(got))
	}
	if after := groupRight(); after != before {
		t.Fatalf("the owning group may %s after the mask widened, and %s before", permWords(after), permWords(before))
	}
}

// watchedFolder is a kept folder with a watch running on it, its users, and the end of the watch's run.
func watchedFolder(t *testing.T) (string, []store.KeptUser, *Watcher, context.CancelFunc, <-chan struct{}) {
	t.Helper()
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "music")
	if err := os.Mkdir(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false,
		"mesh-lidarr:61001:read-write:container")); err != nil {
		t.Fatal(err)
	}
	rec, _ := keptRecord(lib)
	w, err := NewWatcher()
	if err != nil {
		t.Skipf("no inotify here: %v", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	ended := make(chan struct{})
	go func() { w.Run(ctx); close(ended) }()
	// The watch ends before the test's record of kept folders is put back.
	t.Cleanup(func() { cancel(); <-ended })
	w.Set(map[string][]store.KeptUser{lib: rec.Want})
	return lib, rec.Want, w, cancel, ended
}

// Review L4: a watch whose folder goes is let go (IN_IGNORED).
func TestTheWatchOfAFolderRemovedIsLetGo(t *testing.T) {
	lib, _, w, _, _ := watchedFolder(t)
	album := filepath.Join(lib, "album")
	if err := os.Mkdir(album, 0o755); err != nil {
		t.Fatal(err)
	}
	waitFor(t, "the folder made is watched", func() bool { return w.watched() == 2 })
	if err := os.Remove(album); err != nil {
		t.Fatal(err)
	}
	waitFor(t, "the watch of a folder removed is let go", func() bool { return w.watched() == 1 })
}

// Review L4: notice lost re-arms every watch, and the record says the watch is whole again.
func TestNoticeLostReArmsTheWatch(t *testing.T) {
	lib, _, w, _, _ := watchedFolder(t)
	w.lost()
	if w.watched() != 1 {
		t.Fatalf("notice lost did not re-arm the watch: %d watched", w.watched())
	}
	if r, _ := keptRecord(lib); r.OutOfWatches {
		t.Error("a re-armed watch is still said out of watches")
	}
	late := filepath.Join(lib, "late")
	f, _ := os.OpenFile(late, os.O_CREATE|os.O_WRONLY, 0o644)
	f.Close()
	waitFor(t, "the re-armed watch repairs", func() bool { return effectiveOn(t, late, radarr)&6 == 6 })
}

// Review L4: users given while notices are heard are read under the lock (run with -race).
func TestUsersGivenWhileNoticesAreHeardAreReadUnderTheLock(t *testing.T) {
	lib, users, w, _, _ := watchedFolder(t)
	stop := make(chan struct{})
	done := make(chan struct{})
	go func() {
		defer close(done)
		for {
			select {
			case <-stop:
				return
			default:
				w.Set(map[string][]store.KeptUser{lib: users})
			}
		}
	}()
	for i := 0; i < 50; i++ {
		_ = os.WriteFile(filepath.Join(lib, fmt.Sprintf("f%d", i)), nil, 0o644)
	}
	waitFor(t, "the last file is repaired", func() bool {
		return effectiveOn(t, filepath.Join(lib, "f49"), radarr)&6 == 6
	})
	close(stop)
	<-done
}

// Review L4: the watch ends with its context, however quiet the machine.
func TestTheWatchEndsWithItsContext(t *testing.T) {
	_, _, _, cancel, ended := watchedFolder(t)
	cancel()
	select {
	case <-ended:
	case <-time.After(2 * time.Second):
		t.Fatal("the watch did not end with its context")
	}
}

// Review L6: a broken folder's words carry no path.
func TestABrokenFoldersWordsCarryNoPath(t *testing.T) {
	dir := t.TempDir()
	keeping(t, nil, nil)
	file := filepath.Join(dir, "notafolder")
	if err := os.WriteFile(file, nil, 0o644); err != nil {
		t.Fatal(err)
	}
	for _, path := range []string{file, filepath.Join(file, "below")} {
		v := AccessVerdict(context.Background(), path, false, true)
		if v.Kind != VerdictBroken || strings.Contains(v.Why, "/") {
			t.Errorf("%s: %+v", path, v)
		}
	}
}

// Review H2: a systemd .mount unit says where a disk is meant to be, as fstab does; and a mount point seen mounted
// once is remembered, so a folder below it is a missing disk on first sight when it is gone.
func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) {
	units := t.TempDir()
	unit := "[Unit]\nDescription=pool\n\n[Mount]\nWhat=/dev/disk/by-label/pool\nWhere=/srv/pool\nType=ext4\n"
	if err := os.WriteFile(filepath.Join(units, "srv-pool.mount"), []byte(unit), 0o644); err != nil {
		t.Fatal(err)
	}
	if got := mountUnitTargets([]string{units}); len(got) != 1 || got[0] != "/srv/pool" {
		t.Fatalf("read %v from a .mount unit", got)
	}

	dir := t.TempDir()
	disk := filepath.Join(dir, "disk")
	if err := os.Mkdir(disk, 0o755); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{disk}, nil)
	_ = placedOn(context.Background(), disk, disk, "", nil) // seen mounted once
	MountPoints = func() []string { return []string{"/"} }
	if pl := placedOn(context.Background(), filepath.Join(disk, "films"), disk, "", nil); !pl.Away {
		t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk")
	}
}

// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset.
func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) {
	t.Helper()
	dir := aclFolder(t)
	dataset = filepath.Join(dir, "tank")
	if err := os.Mkdir(dataset, 0o755); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{dataset}, nil)
	lib = filepath.Join(dataset, "movies")
	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
		users...)); err != nil {
		t.Fatal(err)
	}
	return dataset, lib
}

// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount
// point is at the path: its writes would land on the filesystem below (§9).
func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) {
	_, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
	MountPoints = func() []string { return []string{"/"} }
	var ran []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		ran = append(ran, name+" "+strings.Join(args, " "))
		if len(args) > 0 && args[0] == "container" {
			return "", errors.New("no such container")
		}
		return "", nil
	}
	d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+
		`,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`)
	report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
	if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
		t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o)
	}
	for _, r := range ran {
		if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") {
			t.Fatalf("the container was started: %s", r)
		}
	}
	if !WaitingOnAccess(d)["radarr.app"] {
		t.Error("liveness is not told the container waits on its access")
	}
}

// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until
// the drop walk can run; then it is forgotten.
func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) {
	dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
	other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}`
	if _, _, err := applyAccesses(t, store.State{}, other); err != nil {
		t.Fatal(err)
	}
	MountPoints = func() []string { return []string{"/"} }
	ReconcileKept(context.Background())
	left := Leftovers()
	if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") ||
		strings.Contains(left[0].Why, dataset) {
		t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left)
	}
	MountPoints = func() []string { return []string{"/", dataset} }
	ReconcileKept(context.Background())
	if left := Leftovers(); len(left) != 0 {
		t.Fatalf("said after the drop ran: %+v", left)
	}
	if _, had := keptRecord(lib); had {
		t.Error("a folder nobody uses was not forgotten once its entries were out")
	}
	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
		t.Error("the entries were not taken out")
	}
}

// Review L3: a disk mounted over a folder taken below it on a warrant leaves the mesh's entries on that data
// recorded and said; once that filesystem holds the folder again, they are taken out.
func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	pool := filepath.Join(dir, "pool")
	lib := filepath.Join(pool, "media")
	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	done, _, err := TakeFolder(context.Background(), declare(t, a), "radarr.media", lib, "w-1", "op", noServices)
	if err != nil {
		t.Fatal(err)
	}
	<-done
	film := filepath.Join(lib, "film")
	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
		t.Fatal(err)
	}
	_ = unix.Removexattr(film, aclAccessXattr)
	MountPoints = func() []string { return []string{"/", pool} }
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") {
		t.Fatalf("entries left on the data taken below are not said: %+v", left)
	}
	// The pool goes: the filesystem below holds the folder again, and the entries left there are taken out.
	MountPoints = func() []string { return []string{"/"} }
	ReconcileKept(context.Background())
	if _, still := func() (uint16, bool) {
		a, _, _ := readACLs(t, filepath.Join(lib, "old"))
		return a.perm(aclUser, radarr)
	}(); still {
		t.Fatal("the entries left on the filesystem that came back were not taken out")
	}
	if left := Leftovers(); len(left) != 0 {
		t.Fatalf("still said after they were taken out: %+v", left)
	}
}

// Review (third) M: a folder kept only because it was empty leaves nothing said when a disk mounts over it — its
// entries sit on an empty folder hidden under the mount and grant nothing on data.
func TestAFolderKeptEmptyLeavesNoLeftoverWhenADiskMountsOverIt(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	pool := filepath.Join(dir, "pool")
	lib := filepath.Join(pool, "media")
	if err := os.MkdirAll(lib, 0o755); err != nil {
		t.Fatal(err)
	}
	a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
		t.Fatal(err)
	}
	MountPoints = func() []string { return []string{"/", pool} }
	report, _, err := applyAccesses(t, store.State{}, a)
	if err != nil {
		t.Fatal(err)
	}
	if left := Leftovers(); len(left) != 0 {
		t.Fatalf("a folder kept while empty is said as leftover entries for ever: %+v", left)
	}
	if rec, _ := keptRecord(lib); len(rec.Left) != 0 {
		t.Errorf("entries on the hidden empty folder are recorded: %+v", rec.Left)
	}
	if o := outcomeOf(report, "radarr.media"); !strings.Contains(o.Detail, "forgotten") {
		t.Errorf("the forgetting is not said: %+v", o)
	}
}

// Review (third) low: a record written before it kept the accesses' ids is said too, under a name with no path.
func TestALeftoverWithNoAccessIdsIsSaidUnderAPathFreeName(t *testing.T) {
	keeping(t, nil, nil)
	path := "/srv/media/old"
	_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
		f.Kept, f.Users = true, []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}}
		f.Act = &store.KeptAct{State: "waiting"}
		return true
	})
	left := Leftovers()
	if len(left) != 1 || left[0].Module != "radarr" || strings.Contains(left[0].Why, path) ||
		strings.Contains(left[0].Access, "/") {
		t.Fatalf("a record without ids is not said, path-free: %+v", left)
	}
}

// placedPool is a folder of a library on a pool the test says is mounted, with the pool's device name and filesystem
// id as the test sets them, taken under a warrant.
func placedPool(t *testing.T, source, fsid *string) (pool, lib, taken string) {
	t.Helper()
	dir := aclFolder(t)
	pool = filepath.Join(dir, "pool")
	lib = filepath.Join(pool, "media")
	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{pool}, nil)
	wasSource, wasFsid := MountSource, MountFsid
	MountSource = func(point string) (string, string) {
		if point == pool {
			return *source, "ext4"
		}
		return "", ""
	}
	MountFsid = func(point string) string {
		if point == pool {
			return *fsid
		}
		return ""
	}
	t.Cleanup(func() { MountSource, MountFsid = wasSource, wasFsid })
	notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil {
		t.Fatal(err)
	}
	done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices)
	if err != nil {
		t.Fatal(err)
	}
	<-done
	taken = accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container")
	if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" {
		t.Fatal("the folder taken waits on its own filesystem")
	}
	return pool, lib, taken
}

// Review (third) low: another filesystem mounted at the same point is not the one the warrant was for.
func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) {
	source, fsid := "/dev/sdb1", "0a0b0c0d01020304"
	_, lib, taken := placedPool(t, &source, &fsid)
	fsid = "1111222233334444" // another filesystem mounted at the same point
	report, _, err := applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("another filesystem at the same mount point was kept on the old warrant: %+v", o)
	}
	if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep {
		t.Fatalf("the look judges it %s", v.Kind)
	}
}

// Review (fourth): the same filesystem whose device the kernel renamed across a reboot (/dev/sdb1 to /dev/sdc1, one
// filesystem id) is the same filesystem: still taken, still kept, and none of its entries walked away.
func TestADeviceRenamedIsTheSameFilesystem(t *testing.T) {
	source, fsid := "/dev/sdb1", "0a0b0c0d01020304"
	_, lib, taken := placedPool(t, &source, &fsid)
	source = "/dev/sdc1"
	report, _, err := applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" {
		t.Fatalf("a device renamed reads as another filesystem: %+v", o)
	}
	ReconcileKept(context.Background())
	if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 {
		t.Fatalf("the library's entries were walked away: the identity may %s", permWords(got))
	}
	if left := Leftovers(); len(left) != 0 {
		t.Fatalf("a device renamed leaves entries said as left: %+v", left)
	}
}

// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on;
// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked.
func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	pool := filepath.Join(dir, "pool")
	lib := filepath.Join(pool, "media")
	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
		t.Fatal(err)
	}
	notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
	if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil {
		t.Fatal(err)
	}
	done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices)
	if err != nil {
		t.Fatal(err)
	}
	<-done
	before := AccessVerdict(context.Background(), lib, true, true)
	film := filepath.Join(lib, "film")
	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
		t.Fatal(err)
	}
	_ = unix.Removexattr(film, aclAccessXattr)
	MountPoints = func() []string { return []string{"/", pool} }
	taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container")
	report, _, err := applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o)
	}
	// And on the next apply too, its record now on the new filesystem.
	report, _, _ = applyAccesses(t, store.State{}, taken)
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("the second apply kept it on the old warrant: %+v", o)
	}
	v := AccessVerdict(context.Background(), lib, true, true)
	if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") {
		t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before)
	}
	ReconcileKept(context.Background())
	if got := effectiveOn(t, film, radarr); got != 0 {
		t.Fatal("the data on the new filesystem was walked")
	}
}

// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile.
func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) {
	was := statBound
	statBound = 20 * time.Millisecond
	defer func() { statBound = was }()
	release := make(chan struct{})
	defer close(release)
	var started int32
	var mu sync.Mutex
	hang := func(string) (os.FileInfo, error) {
		mu.Lock()
		started++
		mu.Unlock()
		<-release
		return nil, nil
	}
	for i := 0; i < 5; i++ {
		if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) {
			t.Fatalf("a hung read answered: %v", err)
		}
	}
	mu.Lock()
	defer mu.Unlock()
	if started != 1 {
		t.Fatalf("%d reads of one path were started while the first hung", started)
	}
}

// Review (fourth): entries left on the filesystem the folder is on now are never walked away — merged back into
// the folder's users when it is kept there, held while it is not.
func TestEntriesLeftOnTheFilesystemTheFolderIsOnAreNeverWalkedAway(t *testing.T) {
	source, fsid := "/dev/sdb1", "0a0b0c0d01020304"
	pool, lib, _ := placedPool(t, &source, &fsid)
	rec, _ := keptRecord(lib)
	users := rec.Users
	for _, kept := range []bool{false, true} {
		_ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool {
			f.Kept, f.Users = kept, nil
			f.Left = []store.KeptLeft{{Mount: pool, Placed: f.Placed, Users: users}}
			return true
		})
		ReconcileKept(context.Background())
		if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 {
			t.Fatalf("kept=%v: entries on the folder's own filesystem were walked away", kept)
		}
		r, _ := keptRecord(lib)
		if kept && (len(r.Left) != 0 || len(r.Users) != 1) {
			t.Errorf("kept: not merged back into the folder's users: %+v", r)
		}
		if !kept && len(r.Left) != 1 {
			t.Errorf("not kept: the entries' record was dropped: %+v", r)
		}
	}
}

// Review (fourth): a TakenOn naming a mount point (an unmerged build's record) is not taken on whatever is mounted
// there now.
func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) {
	dir := aclFolder(t)
	keeping(t, nil, nil)
	lib := filepath.Join(dir, "media")
	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
		t.Fatal(err)
	}
	_ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool {
		f.Mount, f.TakenOn = "/", "/"
		return true
	})
	if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep {
		t.Fatalf("an old-form record is read as taken: %s", v.Kind)
	}
}

// poolNamed is a library on a pool the test says is mounted, whose UUID, filesystem id and mountinfo source are as
// the test sets them; the access declared, not taken.
func poolNamed(t *testing.T, uuid, fsid, source *string) (pool, lib, access string) {
	t.Helper()
	dir := aclFolder(t)
	pool = filepath.Join(dir, "pool")
	lib = filepath.Join(pool, "media")
	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
		t.Fatal(err)
	}
	keeping(t, []string{pool}, nil)
	wasUUID, wasFsid, wasSource := MountUUID, MountFsid, MountSource
	MountUUID = func(p string) string { return pick(p == pool, *uuid) }
	MountFsid = func(p string) string { return pick(p == pool, *fsid) }
	MountSource = func(p string) (string, string) { return pick(p == pool, *source), pick(p == pool, "nfs4") }
	t.Cleanup(func() { MountUUID, MountFsid, MountSource = wasUUID, wasFsid, wasSource })
	return pool, lib, accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
}

func pick(yes bool, v string) string {
	if yes {
		return v
	}
	return "x"
}

// takeIt takes the folder of access under a warrant, and says it is kept.
func takeIt(t *testing.T, lib, access string) {
	t.Helper()
	if _, _, err := applyAccesses(t, store.State{}, access); err != nil {
		t.Fatal(err)
	}
	done, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w-1", "op", noServices)
	if err != nil {
		t.Fatal(err)
	}
	<-done
}

// Review (fifth) M: an NFS export (f_fsid 0, no UUID) is named by its source; another export at the same mount
// point is another filesystem, not taken.
func TestAnNFSExportIsNamedByItsSource(t *testing.T) {
	uuid, fsid, source := "", "0000000000000000", "nas:/export/media"
	_, lib, access := poolNamed(t, &uuid, &fsid, &source)
	takeIt(t, lib, access)
	taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1)
	if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" {
		t.Fatal("the export taken waits on its own filesystem")
	}
	source = "other-nas:/export/media"
	report, _, err := applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("another export at the same mount point was kept on the old warrant: %+v", o)
	}
}

// Review (fifth) M: a filesystem that cannot be named is never kept, nor taken: it fails closed, and waits.
func TestAFilesystemThatCannotBeNamedIsNeverKept(t *testing.T) {
	uuid, fsid, source := "", "0000000000000000", ""
	_, lib, access := poolNamed(t, &uuid, &fsid, &source)
	if err := os.RemoveAll(filepath.Join(lib, "old")); err != nil { // empty: it would be kept without asking
		t.Fatal(err)
	}
	report, _, err := applyAccesses(t, store.State{}, access)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
		t.Fatalf("a folder on a filesystem that cannot be named was kept: %+v", o)
	}
	if got := effectiveOn(t, lib, radarr); got != 0 {
		t.Fatal("an entry was added on a filesystem that cannot be named")
	}
	if _, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w", "op", noServices); err == nil {
		t.Fatal("the take act ran on a filesystem that cannot be named")
	}
}

// Review (fifth): xfs's f_fsid follows the device number; its UUID names it first, so a renumbered device is the same
// filesystem.
func TestAFilesystemsUUIDNamesItBeforeItsFsid(t *testing.T) {
	uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1"
	_, lib, access := poolNamed(t, &uuid, &fsid, &source)
	takeIt(t, lib, access)
	taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1)
	fsid, source = "0000082100000000", "/dev/sdc1"
	report, _, err := applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" {
		t.Fatalf("a renumbered device with one UUID reads as another filesystem: %+v", o)
	}
	ReconcileKept(context.Background())
	if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 {
		t.Fatalf("the library's entries were walked away: %s", permWords(got))
	}
}

// Review (fifth): entries left on a filesystem that never comes back are forgotten after LeftFor, said once, and no
// longer said unhealthy.
func TestEntriesLeftOnAFilesystemThatNeverReturnsAreForgotten(t *testing.T) {
	keeping(t, nil, nil)
	path := "/srv/media/gone"
	_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
		f.IDs = []string{"radarr.media"}
		f.Left = []store.KeptLeft{{Mount: "/srv/media", Placed: "fs-000000000000", Fs: "uuid:gone",
			Since: time.Now().Add(-LeftFor - time.Hour),
			Users: []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}}}}
		return true
	})
	if len(Leftovers()) != 1 {
		t.Fatal("the leftover is not said before it expires")
	}
	said := ReconcileKept(context.Background())
	if left := Leftovers(); len(left) != 0 {
		t.Fatalf("a leftover on a filesystem gone for longer than %s is still said: %+v", LeftFor, left)
	}
	if !strings.Contains(strings.Join(said, "\n"), "forgotten") {
		t.Errorf("the forgetting is not said: %v", said)
	}
}

// Review (fifth) low: naming a filesystem whose read hangs holds one read per mount, whoever asks meanwhile.
func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) {
	was, wasBound := MountUUID, statBound
	statBound = 20 * time.Millisecond
	release := make(chan struct{})
	var mu sync.Mutex
	started := 0
	MountUUID = func(string) string {
		mu.Lock()
		started++
		mu.Unlock()
		<-release
		return ""
	}
	defer func() { close(release); flightsDone(t); MountUUID, statBound = was, wasBound }()
	for i := 0; i < 5; i++ {
		if id := PlacedID("/a/dead/mount"); id != "" {
			t.Fatalf("a filesystem that does not answer is named %s", id)
		}
	}
	mu.Lock()
	defer mu.Unlock()
	if started != 1 {
		t.Fatalf("%d reads of one mount were started while the first hung", started)
	}
}

// Review (sixth): a name read that does not answer is unknown, never a move — a folder kept while empty and filled
// since is not asked about again, a warranted one is not walked again, and nothing of either record is reset.
func TestAFilesystemNotNamedThisTimeIsNeverAMove(t *testing.T) {
	uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1"
	_, lib, access := poolNamed(t, &uuid, &fsid, &source)
	// The name's reads hang past their bound while hanging is set; installed once, before anything reads it.
	var hanging atomic.Bool
	release := make(chan struct{})
	named := MountUUID
	MountUUID = func(p string) string {
		if hanging.Load() {
			<-release
		}
		return named(p)
	}
	takeIt(t, lib, access)
	taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1)
	before, _ := keptRecord(lib)

	wasBound := statBound
	statBound = 20 * time.Millisecond
	hanging.Store(true)
	report, _, err := applyAccesses(t, store.State{}, taken)
	hanging.Store(false)
	close(release)
	flightsDone(t)
	statBound = wasBound
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" || !strings.Contains(o.Detail, "could not be named") {
		t.Fatalf("a folder whose filesystem was not named this time is not said so: %+v", o)
	}
	after, _ := keptRecord(lib)
	if !after.Kept || after.Placed != before.Placed || after.TakenOn != before.TakenOn || len(after.Left) != 0 ||
		!sameUsers(after.Users, before.Users) {
		t.Fatalf("the record was reset by a name not read:\nbefore %+v\nafter  %+v", before, after)
	}

	// Named again: kept as it was, with nothing asked and nothing walked.
	waitFor(t, "the hung read returned", func() bool { return PlacedID(filepath.Dir(lib)) == before.Placed })
	report, _, err = applyAccesses(t, store.State{}, taken)
	if err != nil {
		t.Fatal(err)
	}
	if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" {
		t.Fatalf("named again, the folder asks again: %+v", o)
	}
	if r, _ := keptRecord(lib); r.Act == nil || r.Act.Act != "take" {
		t.Errorf("the folder was walked again: %+v", r.Act)
	}
}

// flightsDone waits until no read of a hung test is in flight: it goes on reading the stand-ins after its release.
func flightsDone(t *testing.T) {
	t.Helper()
	waitFor(t, "the hung reads returned", func() bool {
		inFlight.Lock()
		defer inFlight.Unlock()
		return len(inFlight.reads) == 0
	})
}