Hold what mounts a missing disk, and say entries the mesh could not take out (hq ADR 0325 re-review)

A container was still created over an unmounted mount point, so its writes landed on
the filesystem below; a folder moved to another disk was kept on the old warrant; and
entries left by a drop or a move that could not run lived only in a log line. Every
container mounting an access whose disk is away now waits, the warrant holds only for
the filesystem it was carried out on (the wait names the new one, path-free), entries
left on a filesystem are recorded per mount and taken out when it is back, and every
leftover is said as unhealthy until it is. A hung stat is shared per path, and a
widened mask takes from other entries only the bits it adds.

internal/apply/keeper_linux.go

@@ -61,6 +61,12 @@ func ReconcileKept(ctx context.Context) []string {
 		if ctx.Err() != nil {
 			break
 		}
+		if len(f.Left) > 0 {
+			said = append(said, dropLeft(path, f)...)
+			if f, _ = keptRecord(path); f.Path == "" {
+				continue
+			}
+		}
 		if !f.Kept || sameUsers(f.Users, f.Want) {
 			continue
 		}
@@ -69,6 +75,69 @@ func ReconcileKept(ctx context.Context) []string {
 	return said
 }
 
+// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds
+// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they
+// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and
+// said (Leftovers).
+func dropLeft(path string, f store.KeptFolder) []string {
+	lock := folderLock(path)
+	lock.Lock()
+	defer lock.Unlock()
+	holding := ""
+	if _, err := lstatWithin(path); err == nil {
+		holding = mountHolding(path, MountPoints())
+	}
+	var said []string
+	var still []store.KeptLeft
+	var mine []store.KeptUser
+	for _, l := range f.Left {
+		switch {
+		case f.Kept && l.Mount == f.Mount:
+			mine = append(mine, l.Users...)
+		case l.Mount != holding:
+			still = append(still, l)
+		default:
+			drop := map[uint32]bool{}
+			for _, u := range l.Users {
+				drop[u.UID] = true
+			}
+			w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop))
+			if err != nil || w.Failed != "" {
+				still = append(still, l)
+				said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+					"were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed)))
+				continue
+			}
+			said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
+				"were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files))
+		}
+	}
+	_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
+		k.Left = still
+		for _, u := range mine {
+			if _, has := usersOf(k.Users)[u.UID]; !has {
+				k.Users = append(k.Users, u)
+			}
+		}
+		return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0)
+	})
+	return said
+}
+
+func pathlessOr(err error) string {
+	if err == nil {
+		return ""
+	}
+	return pathless(err)
+}
+
+func firstOf(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
+}
+
 func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string {
 	lock := folderLock(path)
 	lock.Lock()
@@ -196,10 +265,15 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra
 	}
 	started := time.Now().UTC()
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-		if !f.Kept || v.moved {
+		if v.moved {
+			*f = movedRecord(*f, v.mount)
+		}
+		if !f.Kept {
 			f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil
 		}
-		f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users
+		f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids
+		// The warrant is for the filesystem it was carried out on (§8).
+		f.TakenOn = v.mount
 		f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started}
 		return true
 	})