Hold what mounts a missing disk, and say entries the mesh could not take out (hq ADR 0325 re-review)

A container was still created over an unmounted mount point, so its writes landed on the filesystem below; a folder moved to another disk was kept on the old warrant; and entries left by a drop or a move that could not run lived only in a log line. Every container mounting an access whose disk is away now waits, the warrant holds only for the filesystem it was carried out on (the wait names the new one, path-free), entries left on a filesystem are recorded per mount and taken out when it is back, and every leftover is said as unhealthy until it is. A hung stat is shared per path, and a widened mask takes from other entries only the bits it adds.
2026-10-12 19:03:02 +02:00
parent 2026ef36f3
commit d2d9fbbd7d
9 changed files with 453 additions and 39 deletions
@@ -2,6 +2,8 @@ package apply

 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@@ -249,10 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	if why := rootOwned(facts.UID, facts.GID); why != "" {
 		return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why}
 	}
+	// The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a
+	// folder moved to another is not taken, and asks again (§8).
+	takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved))
 	switch {
 	case rec.Kept && !pl.Moved:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How}
-	case taken:
+	case takenHere:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved}
 	case facts.Empty:
 		return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved}
@@ -262,6 +267,45 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken
 	return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved}
 }

+// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left
+// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the
+// controller's word is not read for the new one, and nothing else of what was kept there holds.
+func movedRecord(f store.KeptFolder, mount string) store.KeptFolder {
+	left := f.Left
+	if len(f.Users) > 0 && f.Mount != "" {
+		left = addLeft(left, f.Mount, f.Users)
+	}
+	return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn,
+		Left: left}
+}
+
+// addLeft adds users' entries left on mount, once per mount and uid.
+func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft {
+	for i := range left {
+		if left[i].Mount == mount {
+			have := usersOf(left[i].Users)
+			for _, u := range users {
+				if _, had := have[u.UID]; !had {
+					left[i].Users = append(left[i].Users, u)
+				}
+			}
+			return left
+		}
+	}
+	return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)})
+}
+
+// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
+// folder's filesystem is not the one a warrant was given for.
+func PlacedID(mount string) string {
+	if mount == "" {
+		return ""
+	}
+	source, fstype := mountSourceOf(mount)
+	sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype))
+	return "fs-" + hex.EncodeToString(sum[:6])
+}
+
 // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
 // server is down blocks a stat for ever, and one access must never hold the apply.
 var statBound = 5 * time.Second
@@ -270,21 +314,41 @@ var statBound = 5 * time.Second
 var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")

 // lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
-func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) }
-func statWithin(path string) (os.FileInfo, error)  { return within(os.Stat, path) }

-func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
-	type read struct {
-		info os.FileInfo
-		err  error
+func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
+func statWithin(path string) (os.FileInfo, error)  { return within("stat", os.Stat, path) }
+
+// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead
+// mount holds one goroutine per path, never one per apply or look.
+type statRead struct {
+	done chan struct{}
+	info os.FileInfo
+	err  error
+}
+
+var statsInFlight = struct {
+	sync.Mutex
+	reads map[string]*statRead
+}{reads: map[string]*statRead{}}
+
+func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
+	key := op + " " + path
+	statsInFlight.Lock()
+	r, running := statsInFlight.reads[key]
+	if !running {
+		r = &statRead{done: make(chan struct{})}
+		statsInFlight.reads[key] = r
+		go func() {
+			r.info, r.err = stat(path)
+			statsInFlight.Lock()
+			delete(statsInFlight.reads, key)
+			statsInFlight.Unlock()
+			close(r.done)
+		}()
 	}
-	done := make(chan read, 1)
-	go func() {
-		info, err := stat(path)
-		done <- read{info, err}
-	}()
+	statsInFlight.Unlock()
 	select {
-	case r := <-done:
+	case <-r.done:
 		return r.info, r.err
 	case <-time.After(statBound):
 		return nil, errNoAnswer
@@ -364,7 +428,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 		// What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty
 		// below it): judged afresh, the record of what was kept there no longer holds.
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
-			*f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act}
+			*f = movedRecord(*f, v.mount)
 			return had
 		})
 	}
@@ -405,7 +469,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 			return out, nil
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
-			f.Mount, f.Want = v.mount, users
+			f.Mount, f.Want, f.IDs = v.mount, users, want.ids
 			return true
 		})
 		out.Action = "waiting"
@@ -447,7 +511,11 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O
 	var pending string
 	_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
 		first := !f.Kept
-		f.Kept, f.Mount, f.Want = true, v.mount, users
+		f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids
+		if v.how == store.KeptByWarrant && f.TakenOn == "" {
+			// The controller says it was taken, and this engine has no record of where: here, now.
+			f.TakenOn = v.mount
+		}
 		if first {
 			f.How, f.At = v.how, time.Now().UTC()
 			// A folder that held nothing has nothing under it to walk; one the controller says was taken
@@ -537,8 +605,8 @@ func forgetUndeclared(folders map[string]*folderWant) {
 		}
 		_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
 			f.Want = nil
-			// A folder never kept, or kept with nobody's entries left, is forgotten now.
-			return f.Kept && len(f.Users) > 0
+			// A folder never kept, or kept with nobody's entries left anywhere, is forgotten now.
+			return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0
 		})
 	}
 }
@@ -590,7 +658,7 @@ func accessPaths(resources []declaration.Resource, refused map[string]error) map

 // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there
 // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026).
-func mountsMissing(c *declaration.Container, accesses map[string]string) string {
+func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string {
 	for _, v := range c.Volumes {
 		src := mountSource(v)
 		if !strings.HasPrefix(src, "/") {
@@ -603,6 +671,12 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string
 			if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) {
 				return id
 			}
+			// There, and its disk not: what is at the path is the empty mount point on the filesystem below, and
+			// a container's writes would land there (§9).
+			rec, _ := keptRecord(path)
+			if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away {
+				return id
+			}
 		}
 	}
 	return ""
@@ -620,7 +694,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
 		return out
 	}
 	for _, r := range d.Resources {
-		if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" {
+		if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" {
 			out[c.ID] = true
 		}
 	}
@@ -639,9 +713,10 @@ const (
 	VerdictNobody = "nobody"
 )

-// AccessJudgement is a look's judgement of an access's folder, and why, in words.
+// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at
+// it for a wait, with no path (PlacedID).
 type AccessJudgement struct {
-	Kind, Why string
+	Kind, Why, Placed string
 }

 // AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the
@@ -662,7 +737,43 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud
 	case v.kind == folderMake:
 		return AccessJudgement{Kind: VerdictMake}
 	case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken:
-		return AccessJudgement{Kind: VerdictWaitKeep}
+		return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)}
 	}
 	return AccessJudgement{Kind: VerdictKeep}
 }
+
+// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder,
+// why, in words with no path.
+type Leftover struct {
+	Module, Access, Name, Path, Why string
+}
+
+// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names
+// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each
+// is said until its drop walk has run.
+func Leftovers() []Leftover {
+	all, err := KeptFolders.All()
+	if err != nil {
+		return nil
+	}
+	var out []Leftover
+	for path, f := range all {
+		if len(f.IDs) == 0 {
+			continue
+		}
+		module, name := accessName(f.IDs[0])
+		dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting"
+		switch {
+		case dropping:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " +
+				"0325 §8)"})
+		case len(f.Left) > 0:
+			out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " +
+				name + " on a filesystem no longer at its folder could not be removed: they are removed once that " +
+				"filesystem is back (novox/hq ADR 0325 §8)"})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access })
+	return out
+}