Hold what mounts a missing disk, and say entries the mesh could not take out (hq ADR 0325 re-review)

A container was still created over an unmounted mount point, so its writes landed on
the filesystem below; a folder moved to another disk was kept on the old warrant; and
entries left by a drop or a move that could not run lived only in a log line. Every
container mounting an access whose disk is away now waits, the warrant holds only for
the filesystem it was carried out on (the wait names the new one, path-free), entries
left on a filesystem are recorded per mount and taken out when it is back, and every
leftover is said as unhealthy until it is. A hung stat is shared per path, and a
widened mask takes from other entries only the bits it adds.

cmd/mesh-host/kept.go

@@ -32,6 +32,8 @@ type judgedAccess struct {
 	module, id, name, path string
 	taken                  bool
 	users                  []store.KeptUser
+	// leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries).
+	leftover bool
 }
 
 // accessJudge reads every access on every look. Safe for the apply and the looking loop at once.
@@ -152,9 +154,10 @@ func judgeAccess(a judgedAccess) verdict {
 	v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0)
 	switch v.Kind {
 	case apply.VerdictWaitKeep:
-		return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " +
-			"operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess,
-			Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}}
+		return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " +
+			"it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)",
+			waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
+				What: apply.KeepWords(a.name), Placed: v.Placed}}}
 	case apply.VerdictWaitDisk:
 		return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)",
 			waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
@@ -264,13 +267,25 @@ func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
 		read[i] = v
 	}
 
+	// And the entries nobody wants that could not be taken out yet, each under the access that last named its
+	// folder, until its drop walk has run (ADR 0325 §8).
+	for _, l := range apply.Leftovers() {
+		as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path,
+			leftover: true})
+		read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why})
+	}
+
 	j.mu.Lock()
 	defer j.mu.Unlock()
 	changed := false
 	out := make([]link.ResourceHealth, 0, len(as))
 	for i, a := range as {
 		r := read[i]
-		v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
+		kind := link.KindAccess
+		if a.leftover {
+			kind = link.KindKeptEntries
+		}
+		v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path,
 			State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries}
 		if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason {
 			v.Since = was.Since

cmd/mesh-host/kept_test.go

@@ -38,7 +38,9 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) {
 		t.Fatalf("not waiting with one wait: %+v", v)
 	}
 	raw, _ := json.Marshal(v.waits[0])
-	if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` {
+	want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` +
+		`"placed":"` + v.waits[0].Placed + `"}`
+	if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") {
 		t.Fatalf("the wait reads %s", raw)
 	}
 	if strings.Contains(v.reason, dir) {

internal/apply/acl.go

@@ -142,7 +142,7 @@ func (a acl) check() error {
 
 // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask
 // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps
-// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning
+// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning
 // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave.
 // Nothing else is changed.
 func keptACL(a acl, need map[uint32]uint16) acl {
@@ -159,16 +159,17 @@ func keptACL(a acl, need map[uint32]uint16) acl {
 	if i := out.find(aclMask, 0); i >= 0 {
 		old := out[i].Perm
 		out[i].Perm |= widen
-		if out[i].Perm != old {
+		if added := out[i].Perm &^ old; added != 0 {
 			// **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does
 			// not set here were limited by the old mask: a file made at 0644 under a default ACL carries the
-			// owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its
-			// effective right stays exactly as it was.
+			// owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask
+			// now adds cannot keep its effective right without losing that bit: exactly those bits are taken off
+			// it, and nothing else of it changes. An entry holding none of them is not touched at all.
 			for j := range out {
 				e := &out[j]
 				_, ours := need[e.ID]
-				if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) {
-					e.Perm &= old
+				if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 {
+					e.Perm &^= added
 				}
 			}
 		}

internal/apply/apply.go

@@ -1135,11 +1135,11 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
 			return begin(r), err
 		}
 		// Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9).
-		if id := mountsMissing(res, in.accesses); id != "" {
+		if id := mountsMissing(ctx, res, in.accesses); id != "" {
 			out := begin(r)
 			out.Action = "waiting"
-			out.Detail = "not created: it mounts the access " + id + ", whose folder is not there; created once it " +
-				"is (novox/hq ADR 0325 §9)"
+			out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " +
+				"created once it is (novox/hq ADR 0325 §9)"
 			return out, nil
 		}
 	}

internal/apply/keep.go

@@ -2,6 +2,8 @@ package apply
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@@ -249,10 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	if why := rootOwned(facts.UID, facts.GID); why != "" {
 		return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why}
 	}
+	// The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a
+	// folder moved to another is not taken, and asks again (§8).
+	takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved))
 	switch {
 	case rec.Kept && !pl.Moved:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How}
-	case taken:
+	case takenHere:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved}
 	case facts.Empty:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved}
@@ -262,6 +267,45 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved}
 }
 
+// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left
+// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the
+// controller's word is not read for the new one, and nothing else of what was kept there holds.
+func movedRecord(f store.KeptFolder, mount string) store.KeptFolder {
+	left := f.Left
+	if len(f.Users) > 0 && f.Mount != "" {
+		left = addLeft(left, f.Mount, f.Users)
+	}
+	return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn,
+		Left: left}
+}
+
+// addLeft adds users' entries left on mount, once per mount and uid.
+func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft {
+	for i := range left {
+		if left[i].Mount == mount {
+			have := usersOf(left[i].Users)
+			for _, u := range users {
+				if _, had := have[u.UID]; !had {
+					left[i].Users = append(left[i].Users, u)
+				}
+			}
+			return left
+		}
+	}
+	return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)})
+}
+
+// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
+// folder's filesystem is not the one a warrant was given for.
+func PlacedID(mount string) string {
+	if mount == "" {
+		return ""
+	}
+	source, fstype := mountSourceOf(mount)
+	sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype))
+	return "fs-" + hex.EncodeToString(sum[:6])
+}
+
 // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
 // server is down blocks a stat for ever, and one access must never hold the apply.
 var statBound = 5 * time.Second
@@ -270,21 +314,41 @@ var statBound = 5 * time.Second
 var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
 
 // lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
-func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) }
-func statWithin(path string) (os.FileInfo, error)  { return within(os.Stat, path) }
 
-func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
-	type read struct {
-		info os.FileInfo
-		err  error
+func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
+func statWithin(path string) (os.FileInfo, error)  { return within("stat", os.Stat, path) }
+
+// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead
+// mount holds one goroutine per path, never one per apply or look.
+type statRead struct {
+	done chan struct{}
+	info os.FileInfo
+	err  error
+}
+
+var statsInFlight = struct {
+	sync.Mutex
+	reads map[string]*statRead
+}{reads: map[string]*statRead{}}
+
+func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
+	key := op + " " + path
+	statsInFlight.Lock()
+	r, running := statsInFlight.reads[key]
+	if !running {
+		r = &statRead{done: make(chan struct{})}
+		statsInFlight.reads[key] = r
+		go func() {
+			r.info, r.err = stat(path)
+			statsInFlight.Lock()
+			delete(statsInFlight.reads, key)
+			statsInFlight.Unlock()
+			close(r.done)
+		}()
 	}
-	done := make(chan read, 1)
-	go func() {
-		info, err := stat(path)
-		done <- read{info, err}
-	}()
+	statsInFlight.Unlock()
 	select {
-	case r := <-done:
+	case <-r.done:
 		return r.info, r.err
 	case <-time.After(statBound):
 		return nil, errNoAnswer
@@ -364,7 +428,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 		// What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty
 		// below it): judged afresh, the record of what was kept there no longer holds.
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-			*f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act}
+			*f = movedRecord(*f, v.mount)
 			return had
 		})
 	}
@@ -405,7 +469,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 			return out, nil
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
-			f.Mount, f.Want = v.mount, users
+			f.Mount, f.Want, f.IDs = v.mount, users, want.ids
 			return true
 		})
 		out.Action = "waiting"
@@ -447,7 +511,11 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 	var pending string
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
 		first := !f.Kept
-		f.Kept, f.Mount, f.Want = true, v.mount, users
+		f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids
+		if v.how == store.KeptByWarrant && f.TakenOn == "" {
+			// The controller says it was taken, and this engine has no record of where: here, now.
+			f.TakenOn = v.mount
+		}
 		if first {
 			f.How, f.At = v.how, time.Now().UTC()
 			// A folder that held nothing has nothing under it to walk; one the controller says was taken
@@ -537,8 +605,8 @@ func forgetUndeclared(folders map[string]*folderWant) {
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
 			f.Want = nil
-			// A folder never kept, or kept with nobody's entries left, is forgotten now.
-			return f.Kept && len(f.Users) > 0
+			// A folder never kept, or kept with nobody's entries left anywhere, is forgotten now.
+			return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0
 		})
 	}
 }
@@ -590,7 +658,7 @@ func accessPaths(resources []declaration.Resource, refused map[string]error) map
 
 // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there
 // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026).
-func mountsMissing(c *declaration.Container, accesses map[string]string) string {
+func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string {
 	for _, v := range c.Volumes {
 		src := mountSource(v)
 		if !strings.HasPrefix(src, "/") {
@@ -603,6 +671,12 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string
 			if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) {
 				return id
 			}
+			// There, and its disk not: what is at the path is the empty mount point on the filesystem below, and
+			// a container's writes would land there (§9).
+			rec, _ := keptRecord(path)
+			if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away {
+				return id
+			}
 		}
 	}
 	return ""
@@ -620,7 +694,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
 		return out
 	}
 	for _, r := range d.Resources {
-		if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" {
+		if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" {
 			out[c.ID] = true
 		}
 	}
@@ -639,9 +713,10 @@ const (
 	VerdictNobody = "nobody"
 )
 
-// AccessJudgement is a look's judgement of an access's folder, and why, in words.
+// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at
+// it for a wait, with no path (PlacedID).
 type AccessJudgement struct {
-	Kind, Why string
+	Kind, Why, Placed string
 }
 
 // AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the
@@ -662,7 +737,43 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud
 	case v.kind == folderMake:
 		return AccessJudgement{Kind: VerdictMake}
 	case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken:
-		return AccessJudgement{Kind: VerdictWaitKeep}
+		return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)}
 	}
 	return AccessJudgement{Kind: VerdictKeep}
 }
+
+// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder,
+// why, in words with no path.
+type Leftover struct {
+	Module, Access, Name, Path, Why string
+}
+
+// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names
+// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each
+// is said until its drop walk has run.
+func Leftovers() []Leftover {
+	all, err := KeptFolders.All()
+	if err != nil {
+		return nil
+	}
+	var out []Leftover
+	for path, f := range all {
+		if len(f.IDs) == 0 {
+			continue
+		}
+		module, name := accessName(f.IDs[0])
+		dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting"
+		switch {
+		case dropping:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " +
+				"0325 §8)"})
+		case len(f.Left) > 0:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " on a filesystem no longer at its folder could not be removed: they are removed once that " +
+				"filesystem is back (novox/hq ADR 0325 §8)"})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access })
+	return out
+}

internal/apply/keeper_linux.go

@@ -61,6 +61,12 @@ func ReconcileKept(ctx context.Context) []string {
 		if ctx.Err() != nil {
 			break
 		}
+		if len(f.Left) > 0 {
+			said = append(said, dropLeft(path, f)...)
+			if f, _ = keptRecord(path); f.Path == "" {
+				continue
+			}
+		}
 		if !f.Kept || sameUsers(f.Users, f.Want) {
 			continue
 		}
@@ -69,6 +75,69 @@ func ReconcileKept(ctx context.Context) []string {
 	return said
 }
 
+// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds
+// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they
+// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and
+// said (Leftovers).
+func dropLeft(path string, f store.KeptFolder) []string {
+	lock := folderLock(path)
+	lock.Lock()
+	defer lock.Unlock()
+	holding := ""
+	if _, err := lstatWithin(path); err == nil {
+		holding = mountHolding(path, MountPoints())
+	}
+	var said []string
+	var still []store.KeptLeft
+	var mine []store.KeptUser
+	for _, l := range f.Left {
+		switch {
+		case f.Kept && l.Mount == f.Mount:
+			mine = append(mine, l.Users...)
+		case l.Mount != holding:
+			still = append(still, l)
+		default:
+			drop := map[uint32]bool{}
+			for _, u := range l.Users {
+				drop[u.UID] = true
+			}
+			w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop))
+			if err != nil || w.Failed != "" {
+				still = append(still, l)
+				said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+					"were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed)))
+				continue
+			}
+			said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+				"were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files))
+		}
+	}
+	_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
+		k.Left = still
+		for _, u := range mine {
+			if _, has := usersOf(k.Users)[u.UID]; !has {
+				k.Users = append(k.Users, u)
+			}
+		}
+		return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0)
+	})
+	return said
+}
+
+func pathlessOr(err error) string {
+	if err == nil {
+		return ""
+	}
+	return pathless(err)
+}
+
+func firstOf(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
+}
+
 func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string {
 	lock := folderLock(path)
 	lock.Lock()
@@ -196,10 +265,15 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra
 	}
 	started := time.Now().UTC()
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-		if !f.Kept || v.moved {
+		if v.moved {
+			*f = movedRecord(*f, v.mount)
+		}
+		if !f.Kept {
 			f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil
 		}
-		f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users
+		f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids
+		// The warrant is for the filesystem it was carried out on (§8).
+		f.TakenOn = v.mount
 		f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started}
 		return true
 	})

internal/apply/kept_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -861,3 +862,191 @@ func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) {
 		t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk")
 	}
 }
+
+// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset.
+func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) {
+	t.Helper()
+	dir := aclFolder(t)
+	dataset = filepath.Join(dir, "tank")
+	if err := os.Mkdir(dataset, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	keeping(t, []string{dataset}, nil)
+	lib = filepath.Join(dataset, "movies")
+	if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
+		users...)); err != nil {
+		t.Fatal(err)
+	}
+	return dataset, lib
+}
+
+// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount
+// point is at the path: its writes would land on the filesystem below (§9).
+func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) {
+	_, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
+	MountPoints = func() []string { return []string{"/"} }
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if len(args) > 0 && args[0] == "container" {
+			return "", errors.New("no such container")
+		}
+		return "", nil
+	}
+	d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+
+		`,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`)
+	report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
+	if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
+		t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o)
+	}
+	for _, r := range ran {
+		if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") {
+			t.Fatalf("the container was started: %s", r)
+		}
+	}
+	if !WaitingOnAccess(d)["radarr.app"] {
+		t.Error("liveness is not told the container waits on its access")
+	}
+}
+
+// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until
+// the drop walk can run; then it is forgotten.
+func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) {
+	dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
+	other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}`
+	if _, _, err := applyAccesses(t, store.State{}, other); err != nil {
+		t.Fatal(err)
+	}
+	MountPoints = func() []string { return []string{"/"} }
+	ReconcileKept(context.Background())
+	left := Leftovers()
+	if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") ||
+		strings.Contains(left[0].Why, dataset) {
+		t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left)
+	}
+	MountPoints = func() []string { return []string{"/", dataset} }
+	ReconcileKept(context.Background())
+	if left := Leftovers(); len(left) != 0 {
+		t.Fatalf("said after the drop ran: %+v", left)
+	}
+	if _, had := keptRecord(lib); had {
+		t.Error("a folder nobody uses was not forgotten once its entries were out")
+	}
+	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
+		t.Error("the entries were not taken out")
+	}
+}
+
+// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that
+// filesystem holds the folder again, they are taken out.
+func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) {
+	dir := aclFolder(t)
+	keeping(t, nil, nil)
+	pool := filepath.Join(dir, "pool")
+	lib := filepath.Join(pool, "media")
+	if err := os.MkdirAll(lib, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
+	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
+		t.Fatal(err)
+	}
+	film := filepath.Join(lib, "film")
+	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_ = unix.Removexattr(film, aclAccessXattr)
+	MountPoints = func() []string { return []string{"/", pool} }
+	if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
+		t.Fatal(err)
+	}
+	if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") {
+		t.Fatalf("entries left on the filesystem below are not said: %+v", left)
+	}
+	// The pool goes: the filesystem below holds the folder again, and the entries left there are taken out.
+	MountPoints = func() []string { return []string{"/"} }
+	ReconcileKept(context.Background())
+	if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
+		t.Fatal("the entries left on the filesystem that came back were not taken out")
+	}
+	if left := Leftovers(); len(left) != 0 {
+		t.Fatalf("still said after they were taken out: %+v", left)
+	}
+}
+
+// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on;
+// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked.
+func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) {
+	dir := aclFolder(t)
+	keeping(t, nil, nil)
+	pool := filepath.Join(dir, "pool")
+	lib := filepath.Join(pool, "media")
+	if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
+	if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil {
+		t.Fatal(err)
+	}
+	done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices)
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-done
+	before := AccessVerdict(context.Background(), lib, true, true)
+	film := filepath.Join(lib, "film")
+	if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_ = unix.Removexattr(film, aclAccessXattr)
+	MountPoints = func() []string { return []string{"/", pool} }
+	taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container")
+	report, _, err := applyAccesses(t, store.State{}, taken)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
+		t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o)
+	}
+	// And on the next apply too, its record now on the new filesystem.
+	report, _, _ = applyAccesses(t, store.State{}, taken)
+	if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
+		t.Fatalf("the second apply kept it on the old warrant: %+v", o)
+	}
+	v := AccessVerdict(context.Background(), lib, true, true)
+	if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") {
+		t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before)
+	}
+	ReconcileKept(context.Background())
+	if got := effectiveOn(t, film, radarr); got != 0 {
+		t.Fatal("the data on the new filesystem was walked")
+	}
+}
+
+// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile.
+func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) {
+	was := statBound
+	statBound = 20 * time.Millisecond
+	defer func() { statBound = was }()
+	release := make(chan struct{})
+	defer close(release)
+	var started int32
+	var mu sync.Mutex
+	hang := func(string) (os.FileInfo, error) {
+		mu.Lock()
+		started++
+		mu.Unlock()
+		<-release
+		return nil, nil
+	}
+	for i := 0; i < 5; i++ {
+		if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) {
+			t.Fatalf("a hung read answered: %v", err)
+		}
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if started != 1 {
+		t.Fatalf("%d reads of one path were started while the first hung", started)
+	}
+}

internal/link/messages.go

@@ -309,6 +309,11 @@ const KindDirectory = "directory"
 // it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12).
 const KindAccess = "access"
 
+// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet
+// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at
+// a folder. Unhealthy, said under the access that last named it, until the drop walk has run.
+const KindKeptEntries = "kept-entries"
+
 // ReasonUsedAsFound starts the reason of a directory used as found.
 const ReasonUsedAsFound = "used as found:"
 
@@ -445,6 +450,9 @@ type AccessWait struct {
 	Access string `json:"access"`
 	Part   string `json:"part"`
 	What   string `json:"what"`
+	// Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant
+	// given for another filesystem (the folder moved) does not answer it, and the controller asks again.
+	Placed string `json:"placed,omitempty"`
 }
 
 // KeptEntry is one user of a kept folder as its listing says it.

internal/store/kept.go

@@ -59,6 +59,14 @@ type KeptFolder struct {
 	// declared. Where they differ a walk adds or removes exactly the difference.
 	Users []KeptUser `json:"users,omitempty"`
 	Want  []KeptUser `json:"want,omitempty"`
+	// IDs are the accesses that last named the folder: what a leftover is said under once none does.
+	IDs []string `json:"ids,omitempty"`
+	// TakenOn is the mount point the operator's warrant was carried out on (ADR 0325 §8): the controller's word
+	// that the folder was taken holds for that filesystem alone, and a folder moved to another asks again.
+	TakenOn string `json:"taken_on,omitempty"`
+	// Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or
+	// one gone): taken out by a drop walk once that filesystem holds the folder again, said until then.
+	Left []KeptLeft `json:"left,omitempty"`
 	// Act is the last walk under the folder: a take, an add or a drop.
 	Act *KeptAct `json:"act,omitempty"`
 	// Sweep is how far the sweep got (ADR 0325 §10).
@@ -69,6 +77,12 @@ type KeptFolder struct {
 	OutOfWatches bool `json:"out_of_watches,omitempty"`
 }
 
+// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are.
+type KeptLeft struct {
+	Mount string     `json:"mount"`
+	Users []KeptUser `json:"users"`
+}
+
 // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path.
 type KeptAct struct {
 	// Act is take, add or drop.