cmd/mesh-host/kept.go

@@ -32,6 +32,8 @@ type judgedAccess struct {
 	module, id, name, path string
 	taken                  bool
 	users                  []store.KeptUser
+	// leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries).
+	leftover bool
 }
 
 // accessJudge reads every access on every look. Safe for the apply and the looking loop at once.
@@ -152,9 +154,10 @@ func judgeAccess(a judgedAccess) verdict {
 	v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0)
 	switch v.Kind {
 	case apply.VerdictWaitKeep:
-		return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " +
-			"operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess,
-			Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}}
+		return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " +
+			"it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)",
+			waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
+				What: apply.KeepWords(a.name), Placed: v.Placed}}}
 	case apply.VerdictWaitDisk:
 		return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)",
 			waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
@@ -264,13 +267,25 @@ func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
 		read[i] = v
 	}
 
+	// And the entries nobody wants that could not be taken out yet, each under the access that last named its
+	// folder, until its drop walk has run (ADR 0325 §8).
+	for _, l := range apply.Leftovers() {
+		as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path,
+			leftover: true})
+		read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why})
+	}
+
 	j.mu.Lock()
 	defer j.mu.Unlock()
 	changed := false
 	out := make([]link.ResourceHealth, 0, len(as))
 	for i, a := range as {
 		r := read[i]
-		v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
+		kind := link.KindAccess
+		if a.leftover {
+			kind = link.KindKeptEntries
+		}
+		v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path,
 			State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries}
 		if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason {
 			v.Since = was.Since

cmd/mesh-host/kept_test.go

@@ -38,7 +38,9 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) {
 		t.Fatalf("not waiting with one wait: %+v", v)
 	}
 	raw, _ := json.Marshal(v.waits[0])
-	if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` {
+	want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` +
+		`"placed":"` + v.waits[0].Placed + `"}`
+	if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") {
 		t.Fatalf("the wait reads %s", raw)
 	}
 	if strings.Contains(v.reason, dir) {

internal/apply/acl.go

@@ -142,7 +142,7 @@ func (a acl) check() error {
 
 // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask
 // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps
-// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning
+// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning
 // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave.
 // Nothing else is changed.
 func keptACL(a acl, need map[uint32]uint16) acl {
@@ -159,16 +159,17 @@ func keptACL(a acl, need map[uint32]uint16) acl {
 	if i := out.find(aclMask, 0); i >= 0 {
 		old := out[i].Perm
 		out[i].Perm |= widen
-		if out[i].Perm != old {
+		if added := out[i].Perm &^ old; added != 0 {
 			// **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does
 			// not set here were limited by the old mask: a file made at 0644 under a default ACL carries the
-			// owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its
-			// effective right stays exactly as it was.
+			// owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask
+			// now adds cannot keep its effective right without losing that bit: exactly those bits are taken off
+			// it, and nothing else of it changes. An entry holding none of them is not touched at all.
 			for j := range out {
 				e := &out[j]
 				_, ours := need[e.ID]
-				if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) {
-					e.Perm &= old
+				if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 {
+					e.Perm &^= added
 				}
 			}
 		}

internal/apply/apply.go

@@ -1135,11 +1135,11 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
 			return begin(r), err
 		}
 		// Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9).
-		if id := mountsMissing(res, in.accesses); id != "" {
+		if id := mountsMissing(ctx, res, in.accesses); id != "" {
 			out := begin(r)
 			out.Action = "waiting"
-			out.Detail = "not created: it mounts the access " + id + ", whose folder is not there; created once it " +
-				"is (novox/hq ADR 0325 §9)"
+			out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " +
+				"created once it is (novox/hq ADR 0325 §9)"
 			return out, nil
 		}
 	}

internal/apply/keep.go

@@ -2,6 +2,8 @@ package apply
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@@ -249,10 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	if why := rootOwned(facts.UID, facts.GID); why != "" {
 		return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why}
 	}
+	// The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a
+	// folder moved to another is not taken, and asks again (§8).
+	takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved))
 	switch {
 	case rec.Kept && !pl.Moved:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How}
-	case taken:
+	case takenHere:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved}
 	case facts.Empty:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved}
@@ -262,6 +267,45 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved}
 }
 
+// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left
+// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the
+// controller's word is not read for the new one, and nothing else of what was kept there holds.
+func movedRecord(f store.KeptFolder, mount string) store.KeptFolder {
+	left := f.Left
+	if len(f.Users) > 0 && f.Mount != "" {
+		left = addLeft(left, f.Mount, f.Users)
+	}
+	return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn,
+		Left: left}
+}
+
+// addLeft adds users' entries left on mount, once per mount and uid.
+func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft {
+	for i := range left {
+		if left[i].Mount == mount {
+			have := usersOf(left[i].Users)
+			for _, u := range users {
+				if _, had := have[u.UID]; !had {
+					left[i].Users = append(left[i].Users, u)
+				}
+			}
+			return left
+		}
+	}
+	return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)})
+}
+
+// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
+// folder's filesystem is not the one a warrant was given for.
+func PlacedID(mount string) string {
+	if mount == "" {
+		return ""
+	}
+	source, fstype := mountSourceOf(mount)
+	sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype))
+	return "fs-" + hex.EncodeToString(sum[:6])
+}
+
 // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
 // server is down blocks a stat for ever, and one access must never hold the apply.
 var statBound = 5 * time.Second
@@ -270,21 +314,41 @@ var statBound = 5 * time.Second
 var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
 
 // lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
-func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) }
-func statWithin(path string) (os.FileInfo, error)  { return within(os.Stat, path) }
 
-func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
-	type read struct {
-		info os.FileInfo
-		err  error
+func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
+func statWithin(path string) (os.FileInfo, error)  { return within("stat", os.Stat, path) }
+
+// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead
+// mount holds one goroutine per path, never one per apply or look.
+type statRead struct {
+	done chan struct{}
+	info os.FileInfo
+	err  error
+}
+
+var statsInFlight = struct {
+	sync.Mutex
+	reads map[string]*statRead
+}{reads: map[string]*statRead{}}
+
+func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
+	key := op + " " + path
+	statsInFlight.Lock()
+	r, running := statsInFlight.reads[key]
+	if !running {
+		r = &statRead{done: make(chan struct{})}
+		statsInFlight.reads[key] = r
+		go func() {
+			r.info, r.err = stat(path)
+			statsInFlight.Lock()
+			delete(statsInFlight.reads, key)
+			statsInFlight.Unlock()
+			close(r.done)
+		}()
 	}
-	done := make(chan read, 1)
-	go func() {
-		info, err := stat(path)
-		done <- read{info, err}
-	}()
+	statsInFlight.Unlock()
 	select {
-	case r := <-done:
+	case <-r.done:
 		return r.info, r.err
 	case <-time.After(statBound):
 		return nil, errNoAnswer
@@ -364,7 +428,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 		// What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty
 		// below it): judged afresh, the record of what was kept there no longer holds.
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-			*f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act}
+			*f = movedRecord(*f, v.mount)
 			return had
 		})
 	}
@@ -405,7 +469,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 			return out, nil
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
-			f.Mount, f.Want = v.mount, users
+			f.Mount, f.Want, f.IDs = v.mount, users, want.ids
 			return true
 		})
 		out.Action = "waiting"
@@ -447,7 +511,11 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 	var pending string
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
 		first := !f.Kept
-		f.Kept, f.Mount, f.Want = true, v.mount, users
+		f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids
+		if v.how == store.KeptByWarrant && f.TakenOn == "" {
+			// The controller says it was taken, and this engine has no record of where: here, now.
+			f.TakenOn = v.mount
+		}
 		if first {
 			f.How, f.At = v.how, time.Now().UTC()
 			// A folder that held nothing has nothing under it to walk; one the controller says was taken
@@ -537,8 +605,8 @@ func forgetUndeclared(folders map[string]*folderWant) {
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
 			f.Want = nil
-			// A folder never kept, or kept with nobody's entries left, is forgotten now.
-			return f.Kept && len(f.Users) > 0
+			// A folder never kept, or kept with nobody's entries left anywhere, is forgotten now.
+			return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0
 		})
 	}
 }
@@ -590,7 +658,7 @@ func accessPaths(resources []declaration.Resource, refused map[string]error) map
 
 // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there
 // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026).
-func mountsMissing(c *declaration.Container, accesses map[string]string) string {
+func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string {
 	for _, v := range c.Volumes {
 		src := mountSource(v)
 		if !strings.HasPrefix(src, "/") {
@@ -603,6 +671,12 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string
 			if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) {
 				return id
 			}
+			// There, and its disk not: what is at the path is the empty mount point on the filesystem below, and
+			// a container's writes would land there (§9).
+			rec, _ := keptRecord(path)
+			if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away {
+				return id
+			}
 		}
 	}
 	return ""
@@ -620,7 +694,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
 		return out
 	}
 	for _, r := range d.Resources {
-		if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" {
+		if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" {
 			out[c.ID] = true
 		}
 	}
@@ -639,9 +713,10 @@ const (
 	VerdictNobody = "nobody"
 )
 
-// AccessJudgement is a look's judgement of an access's folder, and why, in words.
+// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at
+// it for a wait, with no path (PlacedID).
 type AccessJudgement struct {
-	Kind, Why string
+	Kind, Why, Placed string
 }
 
 // AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the
@@ -662,7 +737,43 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud
 	case v.kind == folderMake:
 		return AccessJudgement{Kind: VerdictMake}
 	case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken:
-		return AccessJudgement{Kind: VerdictWaitKeep}
+		return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)}
 	}
 	return AccessJudgement{Kind: VerdictKeep}
 }
+
+// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder,
+// why, in words with no path.
+type Leftover struct {
+	Module, Access, Name, Path, Why string
+}
+
+// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names
+// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each
+// is said until its drop walk has run.
+func Leftovers() []Leftover {
+	all, err := KeptFolders.All()
+	if err != nil {
+		return nil
+	}
+	var out []Leftover
+	for path, f := range all {
+		if len(f.IDs) == 0 {
+			continue
+		}
+		module, name := accessName(f.IDs[0])
+		dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting"
+		switch {
+		case dropping:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " +
+				"0325 §8)"})
+		case len(f.Left) > 0:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " on a filesystem no longer at its folder could not be removed: they are removed once that " +
+				"filesystem is back (novox/hq ADR 0325 §8)"})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access })
+	return out
+}

internal/apply/keeper_linux.go

@@ -61,6 +61,12 @@ func ReconcileKept(ctx context.Context) []string {
 		if ctx.Err() != nil {
 			break
 		}
+		if len(f.Left) > 0 {
+			said = append(said, dropLeft(path, f)...)
+			if f, _ = keptRecord(path); f.Path == "" {
+				continue
+			}
+		}
 		if !f.Kept || sameUsers(f.Users, f.Want) {
 			continue
 		}
@@ -69,6 +75,69 @@ func ReconcileKept(ctx context.Context) []string {
 	return said
 }
 
+// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds
+// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they
+// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and
+// said (Leftovers).
+func dropLeft(path string, f store.KeptFolder) []string {
+	lock := folderLock(path)
+	lock.Lock()
+	defer lock.Unlock()
+	holding := ""
+	if _, err := lstatWithin(path); err == nil {
+		holding = mountHolding(path, MountPoints())
+	}
+	var said []string
+	var still []store.KeptLeft
+	var mine []store.KeptUser
+	for _, l := range f.Left {
+		switch {
+		case f.Kept && l.Mount == f.Mount:
+			mine = append(mine, l.Users...)
+		case l.Mount != holding:
+			still = append(still, l)
+		default:
+			drop := map[uint32]bool{}
+			for _, u := range l.Users {
+				drop[u.UID] = true
+			}
+			w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop))
+			if err != nil || w.Failed != "" {
+				still = append(still, l)
+				said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+					"were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed)))
+				continue
+			}
+			said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+				"were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files))
+		}
+	}
+	_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
+		k.Left = still
+		for _, u := range mine {
+			if _, has := usersOf(k.Users)[u.UID]; !has {
+				k.Users = append(k.Users, u)
+			}
+		}
+		return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0)
+	})
+	return said
+}
+
+func pathlessOr(err error) string {
+	if err == nil {
+		return ""
+	}
+	return pathless(err)
+}
+
+func firstOf(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
+}
+
 func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string {
 	lock := folderLock(path)
 	lock.Lock()
@@ -196,10 +265,15 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra
 	}
 	started := time.Now().UTC()
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-		if !f.Kept || v.moved {
+		if v.moved {
+			*f = movedRecord(*f, v.mount)
+		}
+		if !f.Kept {
 			f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil
 		}
-		f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users
+		f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids
+		// The warrant is for the filesystem it was carried out on (§8).
+		f.TakenOn = v.mount
 		f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started}
 		return true
 	})

internal/apply/kept_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -861,3 +862,191 @@ func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) {
 		t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk")
 	}
 }
+
+// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset.
+func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) {
+	t.Helper()
+	dir := aclFolder(t)
+	dataset = filepath.Join(dir, "tank")
+	if err := os.Mkdir(dataset, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	keeping(t, []string{dataset}, nil)
+	lib = filepath.Join(dataset, "movies")
+	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
+		users...)); err != nil {
+		t.Fatal(err)
+	}
+	return dataset, lib
+}
+
+// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount
+// point is at the path: its writes would land on the filesystem below (§9).
+func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) {
+	_, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
+	MountPoints = func() []string { return []string{"/"} }
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if len(args) > 0 && args[0] == "container" {
+			return "", errors.New("no such container")
+		}
+		return "", nil
+	}
+	d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+
+		`,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`)
+	report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
+		t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o)
+	}
+	for _, r := range ran {
+		if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") {
+			t.Fatalf("the container was started: %s", r)
+		}
+	}
+	if !WaitingOnAccess(d)["radarr.app"] {
+		t.Error("liveness is not told the container waits on its access")
+	}
+}
+
+// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until
+// the drop walk can run; then it is forgotten.
+func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) {
+	dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
+	other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}`
+	if _, _, err := applyAccesses(t, store.State{}, other); err != nil {
+		t.Fatal(err)
+	}
+	MountPoints = func() []string { return []string{"/"} }
+	ReconcileKept(context.Background())
+	left := Leftovers()
+	if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") ||
+		strings.Contains(left[0].Why, dataset) {
+		t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left)
+	}
+	MountPoints = func() []string { return []string{"/", dataset} }
+	ReconcileKept(context.Background())
+	if left := Leftovers(); len(left) != 0 {
+		t.Fatalf("said after the drop ran: %+v", left)
+	}
+	if _, had := keptRecord(lib); had {
+		t.Error("a folder nobody uses was not forgotten once its entries were out")
+	}
+	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
+		t.Error("the entries were not taken out")
+	}
+}
+
+// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that
+// filesystem holds the folder again, they are taken out.
+func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) {
+	dir := aclFolder(t)
+	keeping(t, nil, nil)
+	pool := filepath.Join(dir, "pool")
+	lib := filepath.Join(pool, "media")
+	if err := os.MkdirAll(lib, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
+	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
+		t.Fatal(err)
+	}
+	film := filepath.Join(lib, "film")
+	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_ = unix.Removexattr(film, aclAccessXattr)
+	MountPoints = func() []string { return []string{"/", pool} }
+	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
+		t.Fatal(err)
+	}
+	if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") {
+		t.Fatalf("entries left on the filesystem below are not said: %+v", left)
+	}
+	// The pool goes: the filesystem below holds the folder again, and the entries left there are taken out.
+	MountPoints = func() []string { return []string{"/"} }
+	ReconcileKept(context.Background())
+	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
+		t.Fatal("the entries left on the filesystem that came back were not taken out")
+	}
+	if left := Leftovers(); len(left) != 0 {
+		t.Fatalf("still said after they were taken out: %+v", left)
+	}
+}
+
+// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on;
+// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked.
+func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) {
+	dir := aclFolder(t)
+	keeping(t, nil, nil)
+	pool := filepath.Join(dir, "pool")
+	lib := filepath.Join(pool, "media")
+	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
+	if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil {
+		t.Fatal(err)
+	}
+	done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices)
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-done
+	before := AccessVerdict(context.Background(), lib, true, true)
+	film := filepath.Join(lib, "film")
+	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_ = unix.Removexattr(film, aclAccessXattr)
+	MountPoints = func() []string { return []string{"/", pool} }
+	taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container")
+	report, _, err := applyAccesses(t, store.State{}, taken)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
+		t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o)
+	}
+	// And on the next apply too, its record now on the new filesystem.
+	report, _, _ = applyAccesses(t, store.State{}, taken)
+	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
+		t.Fatalf("the second apply kept it on the old warrant: %+v", o)
+	}
+	v := AccessVerdict(context.Background(), lib, true, true)
+	if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") {
+		t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before)
+	}
+	ReconcileKept(context.Background())
+	if got := effectiveOn(t, film, radarr); got != 0 {
+		t.Fatal("the data on the new filesystem was walked")
+	}
+}
+
+// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile.
+func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) {
+	was := statBound
+	statBound = 20 * time.Millisecond
+	defer func() { statBound = was }()
+	release := make(chan struct{})
+	defer close(release)
+	var started int32
+	var mu sync.Mutex
+	hang := func(string) (os.FileInfo, error) {
+		mu.Lock()
+		started++
+		mu.Unlock()
+		<-release
+		return nil, nil
+	}
+	for i := 0; i < 5; i++ {
+		if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) {
+			t.Fatalf("a hung read answered: %v", err)
+		}
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if started != 1 {
+		t.Fatalf("%d reads of one path were started while the first hung", started)
+	}
+}

internal/link/messages.go

@@ -309,6 +309,11 @@ const KindDirectory = "directory"
 // it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12).
 const KindAccess = "access"
 
+// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet
+// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at
+// a folder. Unhealthy, said under the access that last named it, until the drop walk has run.
+const KindKeptEntries = "kept-entries"
+
 // ReasonUsedAsFound starts the reason of a directory used as found.
 const ReasonUsedAsFound = "used as found:"
 
@@ -445,6 +450,9 @@ type AccessWait struct {
 	Access string `json:"access"`
 	Part   string `json:"part"`
 	What   string `json:"what"`
+	// Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant
+	// given for another filesystem (the folder moved) does not answer it, and the controller asks again.
+	Placed string `json:"placed,omitempty"`
 }
 
 // KeptEntry is one user of a kept folder as its listing says it.

internal/store/kept.go

@@ -59,6 +59,14 @@ type KeptFolder struct {
 	// declared. Where they differ a walk adds or removes exactly the difference.
 	Users []KeptUser `json:"users,omitempty"`
 	Want  []KeptUser `json:"want,omitempty"`
+	// IDs are the accesses that last named the folder: what a leftover is said under once none does.
+	IDs []string `json:"ids,omitempty"`
+	// TakenOn is the mount point the operator's warrant was carried out on (ADR 0325 §8): the controller's word
+	// that the folder was taken holds for that filesystem alone, and a folder moved to another asks again.
+	TakenOn string `json:"taken_on,omitempty"`
+	// Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or
+	// one gone): taken out by a drop walk once that filesystem holds the folder again, said until then.
+	Left []KeptLeft `json:"left,omitempty"`
 	// Act is the last walk under the folder: a take, an add or a drop.
 	Act *KeptAct `json:"act,omitempty"`
 	// Sweep is how far the sweep got (ADR 0325 §10).
@@ -69,6 +77,12 @@ type KeptFolder struct {
 	OutOfWatches bool `json:"out_of_watches,omitempty"`
 }
 
+// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are.
+type KeptLeft struct {
+	Mount string     `json:"mount"`
+	Users []KeptUser `json:"users"`
+}
+
 // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path.
 type KeptAct struct {
 	// Act is take, add or drop.