diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 5f50759..dbd31d9 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -203,6 +203,8 @@ type folderVerdict struct { err error // moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh. moved bool + // unnamed is a folder whose filesystem cannot be named: never kept, nor taken. + unnamed bool } // assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then @@ -244,9 +246,10 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if pl.Away { return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} } - // Another filesystem at the same mount point (another pool, another disk) is a move too. + // Another filesystem at the same mount point (another pool, another disk) is a move too. A record from an earlier + // build names its filesystem another way, and is not compared: the next keep names it this way. here := PlacedID(pl.Mount) - moved := pl.Moved || (rec.Placed != "" && rec.Mount == pl.Mount && rec.Placed != here) + moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here) facts, err := readFolder(path) if err != nil { return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} @@ -254,6 +257,11 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } + // A filesystem that cannot be named is never kept: what is there could be anyone's (§8). + if here == "" { + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved, unnamed: true, + why: "its filesystem cannot be named (no UUID, no filesystem id, no source)"} + } // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a // folder moved to another is not taken, and asks again (§8). takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved)) @@ -279,14 +287,21 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { left := f.Left if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" { - left = addLeft(left, f.Mount, f.Placed, f.Users) + left = addLeft(left, f.Mount, f.Placed, f.Fs, f.Users) } - return store.KeptFolder{Path: f.Path, Mount: mount, Placed: PlacedID(mount), Want: f.Want, IDs: f.IDs, + placed, fs := placeOf(mount) + return store.KeptFolder{Path: f.Path, Mount: mount, Placed: placed, Fs: fs, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, Left: left} } +// placeOf is the name of the filesystem at mount, and what it was made from. +func placeOf(mount string) (placed, fs string) { + fs = filesystemKey(mount) + return placedFrom(mount, fs), fs +} + // addLeft adds users' entries left on mount, once per mount and uid. -func addLeft(left []store.KeptLeft, mount, placed string, users []store.KeptUser) []store.KeptLeft { +func addLeft(left []store.KeptLeft, mount, placed, fs string, users []store.KeptUser) []store.KeptLeft { for i := range left { if left[i].Mount == mount && left[i].Placed == placed { have := usersOf(left[i].Users) @@ -298,35 +313,57 @@ func addLeft(left []store.KeptLeft, mount, placed string, users []store.KeptUser return left } } - return append(left, store.KeptLeft{Mount: mount, Placed: placed, Users: append([]store.KeptUser(nil), users...)}) + return append(left, store.KeptLeft{Mount: mount, Placed: placed, Fs: fs, Since: time.Now().UTC(), + Users: append([]store.KeptUser(nil), users...)}) } // PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the -// folder's filesystem is not the one a warrant was given for. It is built from what stays the same across reboots -// for one filesystem — the kernel's filesystem id (statfs f_fsid: ext4 and xfs derive it from the filesystem's -// UUID, ZFS from the dataset), the filesystem's type and its mount point — and never from the device name -// mountinfo gives as its source, which a reboot may renumber (/dev/sdb1 becoming /dev/sdc1). -func PlacedID(mount string) string { - if mount == "" { +// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder +// on it is then never kept (it fails closed, and waits). +func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) } + +// placedFrom is PlacedID from a filesystem key already read. +func placedFrom(mount, key string) string { + if mount == "" || key == "" { return "" } - _, fstype := mountSourceOf(mount) - fsid := fsidWithin(mount) - sum := sha256.Sum256([]byte(mount + "\x00" + fsid + "\x00" + fstype)) + sum := sha256.Sum256([]byte(mount + "\x00" + key)) return "fs-" + hex.EncodeToString(sum[:6]) } -// fsidWithin is MountFsid within statBound: empty when it does not answer, so a mount that is down is named by its -// point and type alone, never held on. -func fsidWithin(mount string) string { - done := make(chan string, 1) - go func() { done <- MountFsid(mount) }() - select { - case id := <-done: - return id - case <-time.After(statBound): +// filesystemKey is what names the filesystem at mount across reboots, in this order: +// +// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in +// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first; +// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its +// UUID; +// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own +// source string (both report f_fsid 0). +// +// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of +// these answers within statBound: then nothing is kept there. Read once per mount at a time. +func filesystemKey(mount string) string { + if mount == "" { return "" } + v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) }) + if !ok { + return "" + } + return v.(string) +} + +func readFilesystemKey(mount string) string { + if u := MountUUID(mount); u != "" { + return "uuid:" + u + } + if id := MountFsid(mount); strings.Trim(id, "0") != "" { + return "fsid:" + id + } + if source, fstype := mountSourceOf(mount); source != "" && source != "none" { + return "source:" + fstype + ":" + source + } + return "" } // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose @@ -341,41 +378,57 @@ var errNoAnswer = errors.New("it does not answer: a read of it did not return wi func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) } func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) } -// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead -// mount holds one goroutine per path, never one per apply or look. -type statRead struct { +// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds +// one goroutine per path, never one per apply or look. +type flight struct { done chan struct{} - info os.FileInfo - err error + v any } -var statsInFlight = struct { +var inFlight = struct { sync.Mutex - reads map[string]*statRead -}{reads: map[string]*statRead{}} + reads map[string]*flight +}{reads: map[string]*flight{}} -func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { - key := op + " " + path - statsInFlight.Lock() - r, running := statsInFlight.reads[key] +// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not +// answer in time. +func boundedOnce(key string, read func() any) (any, bool) { + inFlight.Lock() + f, running := inFlight.reads[key] if !running { - r = &statRead{done: make(chan struct{})} - statsInFlight.reads[key] = r + f = &flight{done: make(chan struct{})} + inFlight.reads[key] = f go func() { - r.info, r.err = stat(path) - statsInFlight.Lock() - delete(statsInFlight.reads, key) - statsInFlight.Unlock() - close(r.done) + f.v = read() + inFlight.Lock() + delete(inFlight.reads, key) + inFlight.Unlock() + close(f.done) }() } - statsInFlight.Unlock() + inFlight.Unlock() select { - case <-r.done: - return r.info, r.err + case <-f.done: + return f.v, true case <-time.After(statBound): + return nil, false + } +} + +func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + type read struct { + info os.FileInfo + err error + } + v, ok := boundedOnce(op+" "+path, func() any { + info, err := stat(path) + return read{info, err} + }) + if !ok { return nil, errNoAnswer } + r := v.(read) + return r.info, r.err } // absPath is an absolute path in words: what pathless takes out. @@ -496,7 +549,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O return out, nil } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Placed, f.Want, f.IDs = v.mount, PlacedID(v.mount), users, want.ids + f.Mount, f.Want, f.IDs = v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) return true }) out.Action = "waiting" @@ -524,7 +578,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O changed, err := keepTop(path, usersOf(users)) if errors.Is(err, ErrNoACL) { _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Placed, f.Want = v.mount, PlacedID(v.mount), users + f.Mount, f.Want = v.mount, users + f.Placed, f.Fs = placeOf(v.mount) return true }) out.Action = "waiting" @@ -538,7 +593,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O var pending string _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { first := !f.Kept - f.Kept, f.Mount, f.Placed, f.Want, f.IDs = true, v.mount, PlacedID(v.mount), users, want.ids + f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) if v.how == store.KeptByWarrant && f.TakenOn == "" { // The controller says it was taken, and this engine has no record of where: here, now. f.TakenOn = PlacedID(v.mount) diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 5c6e028..8e710e2 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -75,6 +75,9 @@ func ReconcileKept(ctx context.Context) []string { return said } +// LeftFor is how long entries left on a filesystem are kept recorded, and said, while it is not back. +var LeftFor = 30 * 24 * time.Hour + // dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds // it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they // are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and @@ -95,7 +98,20 @@ func dropLeft(path string, f store.KeptFolder) []string { var still []store.KeptLeft var mine []store.KeptUser for _, l := range f.Left { - same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed) + // On the folder's own filesystem now: the same name, or a name of an earlier build's form at the same + // mount point (which cannot be compared, and is never walked away). + same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed || l.Fs == "") + back := l.Mount == holding && (l.Placed == "" || l.Placed == holdingID) + if l.Since.IsZero() { + l.Since = time.Now().UTC() + } + if !same && !back && time.Since(l.Since) > LeftFor { + // **A filesystem not back within LeftFor is gone** (a disk replaced, a pool destroyed): what was left + // on it is forgotten, said once, rather than said unhealthy for ever. + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem not back for %s "+ + "are forgotten: it is taken for gone", path, LeftFor)) + continue + } switch { case f.Kept && same: // Left on the filesystem the folder is kept on now: the folder's own entries. @@ -104,7 +120,7 @@ func dropLeft(path string, f store.KeptFolder) []string { // On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's // once it is kept there again. still = append(still, l) - case l.Mount != holding || (l.Placed != "" && l.Placed != holdingID): + case !back: still = append(still, l) default: drop := map[uint32]bool{} @@ -253,6 +269,8 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra return nil, "", errors.New(RootWords(v.why)) case v.kind == folderMake: return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody") + case v.unnamed: + return nil, "", errors.New(v.why + ", so a warrant could name no filesystem it was given for") } lock := folderLock(path) if !lock.TryLock() { @@ -283,7 +301,8 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids // The warrant is for the filesystem it was carried out on (§8), named as a wait names it. - f.Placed, f.TakenOn = PlacedID(v.mount), PlacedID(v.mount) + f.Placed, f.Fs = placeOf(v.mount) + f.TakenOn = f.Placed f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} return true }) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 53239fa..3fab036 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -1229,3 +1229,156 @@ func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) { t.Fatalf("an old-form record is read as taken: %s", v.Kind) } } + +// poolNamed is a library on a pool the test says is mounted, whose UUID, filesystem id and mountinfo source are as +// the test sets them; the access declared, not taken. +func poolNamed(t *testing.T, uuid, fsid, source *string) (pool, lib, access string) { + t.Helper() + dir := aclFolder(t) + pool = filepath.Join(dir, "pool") + lib = filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{pool}, nil) + wasUUID, wasFsid, wasSource := MountUUID, MountFsid, MountSource + MountUUID = func(p string) string { return pick(p == pool, *uuid) } + MountFsid = func(p string) string { return pick(p == pool, *fsid) } + MountSource = func(p string) (string, string) { return pick(p == pool, *source), pick(p == pool, "nfs4") } + t.Cleanup(func() { MountUUID, MountFsid, MountSource = wasUUID, wasFsid, wasSource }) + return pool, lib, accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") +} + +func pick(yes bool, v string) string { + if yes { + return v + } + return "x" +} + +// takeIt takes the folder of access under a warrant, and says it is kept. +func takeIt(t *testing.T, lib, access string) { + t.Helper() + if _, _, err := applyAccesses(t, store.State{}, access); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done +} + +// Review (fifth) M: an NFS export (f_fsid 0, no UUID) is named by its source; another export at the same mount +// point is another filesystem, not taken. +func TestAnNFSExportIsNamedByItsSource(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "nas:/export/media" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { + t.Fatal("the export taken waits on its own filesystem") + } + source = "other-nas:/export/media" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("another export at the same mount point was kept on the old warrant: %+v", o) + } +} + +// Review (fifth) M: a filesystem that cannot be named is never kept, nor taken: it fails closed, and waits. +func TestAFilesystemThatCannotBeNamedIsNeverKept(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + if err := os.RemoveAll(filepath.Join(lib, "old")); err != nil { // empty: it would be kept without asking + t.Fatal(err) + } + report, _, err := applyAccesses(t, store.State{}, access) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder on a filesystem that cannot be named was kept: %+v", o) + } + if got := effectiveOn(t, lib, radarr); got != 0 { + t.Fatal("an entry was added on a filesystem that cannot be named") + } + if _, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w", "op", noServices); err == nil { + t.Fatal("the take act ran on a filesystem that cannot be named") + } +} + +// Review (fifth): xfs's f_fsid follows the device number; its UUID names it first, so a renumbered device is the same +// filesystem. +func TestAFilesystemsUUIDNamesItBeforeItsFsid(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + fsid, source = "0000082100000000", "/dev/sdc1" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("a renumbered device with one UUID reads as another filesystem: %+v", o) + } + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("the library's entries were walked away: %s", permWords(got)) + } +} + +// Review (fifth): entries left on a filesystem that never comes back are forgotten after LeftFor, said once, and no +// longer said unhealthy. +func TestEntriesLeftOnAFilesystemThatNeverReturnsAreForgotten(t *testing.T) { + keeping(t, nil, nil) + path := "/srv/media/gone" + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.IDs = []string{"radarr.media"} + f.Left = []store.KeptLeft{{Mount: "/srv/media", Placed: "fs-000000000000", Fs: "uuid:gone", + Since: time.Now().Add(-LeftFor - time.Hour), + Users: []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}}}} + return true + }) + if len(Leftovers()) != 1 { + t.Fatal("the leftover is not said before it expires") + } + said := ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a leftover on a filesystem gone for longer than %s is still said: %+v", LeftFor, left) + } + if !strings.Contains(strings.Join(said, "\n"), "forgotten") { + t.Errorf("the forgetting is not said: %v", said) + } +} + +// Review (fifth) low: naming a filesystem whose read hangs holds one read per mount, whoever asks meanwhile. +func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) { + was, wasBound := MountUUID, statBound + statBound = 20 * time.Millisecond + release := make(chan struct{}) + var mu sync.Mutex + started := 0 + MountUUID = func(string) string { + mu.Lock() + started++ + mu.Unlock() + <-release + return "" + } + defer func() { close(release); MountUUID, statBound = was, wasBound }() + for i := 0; i < 5; i++ { + if id := PlacedID("/a/dead/mount"); id != "" { + t.Fatalf("a filesystem that does not answer is named %s", id) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one mount were started while the first hung", started) + } +} diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go index 411162c..692b312 100644 --- a/internal/apply/walk_linux.go +++ b/internal/apply/walk_linux.go @@ -548,3 +548,25 @@ var MountFsid = func(point string) string { } return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1])) } + +// MountUUID is the UUID of the filesystem at point when it is on a block device: the name in /dev/disk/by-uuid of +// the device the mount's own device number is. Empty for one on no block device (NFS, FUSE, ZFS, a btrfs +// subvolume's own device). A variable so a test can stand in for disks it cannot make. +var MountUUID = func(point string) string { + var st unix.Stat_t + if err := unix.Stat(point, &st); err != nil { + return "" + } + const dir = "/dev/disk/by-uuid" + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + for _, e := range entries { + var dev unix.Stat_t + if unix.Stat(dir+"/"+e.Name(), &dev) == nil && dev.Mode&unix.S_IFMT == unix.S_IFBLK && dev.Rdev == st.Dev { + return e.Name() + } + } + return "" +} diff --git a/internal/apply/walk_other.go b/internal/apply/walk_other.go index 1a29965..7d9c79a 100644 --- a/internal/apply/walk_other.go +++ b/internal/apply/walk_other.go @@ -56,3 +56,6 @@ func (*Watcher) Run(context.Context) {} // MountFsid reads no filesystem id where a folder is not kept. var MountFsid = func(string) string { return "" } + +// MountUUID reads no UUID where a folder is not kept. +var MountUUID = func(string) string { return "" } diff --git a/internal/store/kept.go b/internal/store/kept.go index 3dfb6a4..4e9252e 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -64,6 +64,10 @@ type KeptFolder struct { // Placed names the filesystem the folder was kept on (the engine's PlacedID: its mount point, source and type), // so another filesystem mounted at the same point is a move, as another mount point is. Placed string `json:"placed,omitempty"` + // Fs is what Placed was made from beside the mount point: "uuid:", "fsid:" + // or "source::". Kept on this machine only, never said. Absent on a record from an + // earlier build, whose Placed is then not compared. + Fs string `json:"fs,omitempty"` // TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the // controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another // asks again. One that names no filesystem this way (a mount point, from an unmerged build) is not taken. @@ -87,7 +91,11 @@ type KeptLeft struct { // Placed names that filesystem as KeptFolder.Placed does: entries left on the filesystem the folder is on // now are the folder's own, never walked away. Placed string `json:"placed,omitempty"` + Fs string `json:"fs,omitempty"` Users []KeptUser `json:"users"` + // Since is when they were left: a filesystem not back at its mount point within LeftFor is gone, and what was + // left on it is forgotten (said once). + Since time.Time `json:"since,omitzero"` } // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path.