diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index ba742b5..6f9ed3a 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -32,6 +32,8 @@ type judgedAccess struct { module, id, name, path string taken bool users []store.KeptUser + // leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries). + leftover bool } // accessJudge reads every access on every look. Safe for the apply and the looking loop at once. @@ -152,9 +154,10 @@ func judgeAccess(a judgedAccess) verdict { v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0) switch v.Kind { case apply.VerdictWaitKeep: - return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " + - "operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess, - Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}} + return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " + + "it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)", + waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, + What: apply.KeepWords(a.name), Placed: v.Placed}}} case apply.VerdictWaitDisk: return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)", waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, @@ -264,13 +267,25 @@ func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { read[i] = v } + // And the entries nobody wants that could not be taken out yet, each under the access that last named its + // folder, until its drop walk has run (ADR 0325 §8). + for _, l := range apply.Leftovers() { + as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path, + leftover: true}) + read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why}) + } + j.mu.Lock() defer j.mu.Unlock() changed := false out := make([]link.ResourceHealth, 0, len(as)) for i, a := range as { r := read[i] - v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path, + kind := link.KindAccess + if a.leftover { + kind = link.KindKeptEntries + } + v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path, State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries} if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason { v.Since = was.Since diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go index 3bf831e..c5b6817 100644 --- a/cmd/mesh-host/kept_test.go +++ b/cmd/mesh-host/kept_test.go @@ -38,7 +38,9 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { t.Fatalf("not waiting with one wait: %+v", v) } raw, _ := json.Marshal(v.waits[0]) - if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` { + want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` + + `"placed":"` + v.waits[0].Placed + `"}` + if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") { t.Fatalf("the wait reads %s", raw) } if strings.Contains(v.reason, dir) { diff --git a/internal/apply/acl.go b/internal/apply/acl.go index 6fbab09..2b8e8c5 100644 --- a/internal/apply/acl.go +++ b/internal/apply/acl.go @@ -142,7 +142,7 @@ func (a acl) check() error { // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps -// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning +// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. // Nothing else is changed. func keptACL(a acl, need map[uint32]uint16) acl { @@ -159,16 +159,17 @@ func keptACL(a acl, need map[uint32]uint16) acl { if i := out.find(aclMask, 0); i >= 0 { old := out[i].Perm out[i].Perm |= widen - if out[i].Perm != old { + if added := out[i].Perm &^ old; added != 0 { // **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does // not set here were limited by the old mask: a file made at 0644 under a default ACL carries the - // owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its - // effective right stays exactly as it was. + // owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask + // now adds cannot keep its effective right without losing that bit: exactly those bits are taken off + // it, and nothing else of it changes. An entry holding none of them is not touched at all. for j := range out { e := &out[j] _, ours := need[e.ID] - if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) { - e.Perm &= old + if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 { + e.Perm &^= added } } } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 31e4aaa..4440a27 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1135,11 +1135,11 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru return begin(r), err } // Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9). - if id := mountsMissing(res, in.accesses); id != "" { + if id := mountsMissing(ctx, res, in.accesses); id != "" { out := begin(r) out.Action = "waiting" - out.Detail = "not created: it mounts the access " + id + ", whose folder is not there; created once it " + - "is (novox/hq ADR 0325 §9)" + out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " + + "created once it is (novox/hq ADR 0325 §9)" return out, nil } } diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 440f5eb..1a6be6d 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -2,6 +2,8 @@ package apply import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "os" @@ -249,10 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } + // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a + // folder moved to another is not taken, and asks again (§8). + takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved)) switch { case rec.Kept && !pl.Moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} - case taken: + case takenHere: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved} case facts.Empty: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved} @@ -262,6 +267,45 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved} } +// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left +// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the +// controller's word is not read for the new one, and nothing else of what was kept there holds. +func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { + left := f.Left + if len(f.Users) > 0 && f.Mount != "" { + left = addLeft(left, f.Mount, f.Users) + } + return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, + Left: left} +} + +// addLeft adds users' entries left on mount, once per mount and uid. +func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft { + for i := range left { + if left[i].Mount == mount { + have := usersOf(left[i].Users) + for _, u := range users { + if _, had := have[u.UID]; !had { + left[i].Users = append(left[i].Users, u) + } + } + return left + } + } + return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)}) +} + +// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the +// folder's filesystem is not the one a warrant was given for. +func PlacedID(mount string) string { + if mount == "" { + return "" + } + source, fstype := mountSourceOf(mount) + sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype)) + return "fs-" + hex.EncodeToString(sum[:6]) +} + // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose // server is down blocks a stat for ever, and one access must never hold the apply. var statBound = 5 * time.Second @@ -270,21 +314,41 @@ var statBound = 5 * time.Second var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)") // lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own. -func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) } -func statWithin(path string) (os.FileInfo, error) { return within(os.Stat, path) } -func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { - type read struct { - info os.FileInfo - err error +func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) } +func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) } + +// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead +// mount holds one goroutine per path, never one per apply or look. +type statRead struct { + done chan struct{} + info os.FileInfo + err error +} + +var statsInFlight = struct { + sync.Mutex + reads map[string]*statRead +}{reads: map[string]*statRead{}} + +func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + key := op + " " + path + statsInFlight.Lock() + r, running := statsInFlight.reads[key] + if !running { + r = &statRead{done: make(chan struct{})} + statsInFlight.reads[key] = r + go func() { + r.info, r.err = stat(path) + statsInFlight.Lock() + delete(statsInFlight.reads, key) + statsInFlight.Unlock() + close(r.done) + }() } - done := make(chan read, 1) - go func() { - info, err := stat(path) - done <- read{info, err} - }() + statsInFlight.Unlock() select { - case r := <-done: + case <-r.done: return r.info, r.err case <-time.After(statBound): return nil, errNoAnswer @@ -364,7 +428,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O // What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty // below it): judged afresh, the record of what was kept there no longer holds. _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - *f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act} + *f = movedRecord(*f, v.mount) return had }) } @@ -405,7 +469,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O return out, nil } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Want = v.mount, users + f.Mount, f.Want, f.IDs = v.mount, users, want.ids return true }) out.Action = "waiting" @@ -447,7 +511,11 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O var pending string _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { first := !f.Kept - f.Kept, f.Mount, f.Want = true, v.mount, users + f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + if v.how == store.KeptByWarrant && f.TakenOn == "" { + // The controller says it was taken, and this engine has no record of where: here, now. + f.TakenOn = v.mount + } if first { f.How, f.At = v.how, time.Now().UTC() // A folder that held nothing has nothing under it to walk; one the controller says was taken @@ -537,8 +605,8 @@ func forgetUndeclared(folders map[string]*folderWant) { } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { f.Want = nil - // A folder never kept, or kept with nobody's entries left, is forgotten now. - return f.Kept && len(f.Users) > 0 + // A folder never kept, or kept with nobody's entries left anywhere, is forgotten now. + return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0 }) } } @@ -590,7 +658,7 @@ func accessPaths(resources []declaration.Resource, refused map[string]error) map // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026). -func mountsMissing(c *declaration.Container, accesses map[string]string) string { +func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string { for _, v := range c.Volumes { src := mountSource(v) if !strings.HasPrefix(src, "/") { @@ -603,6 +671,12 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) { return id } + // There, and its disk not: what is at the path is the empty mount point on the filesystem below, and + // a container's writes would land there (§9). + rec, _ := keptRecord(path) + if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away { + return id + } } } return "" @@ -620,7 +694,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool { return out } for _, r := range d.Resources { - if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" { + if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" { out[c.ID] = true } } @@ -639,9 +713,10 @@ const ( VerdictNobody = "nobody" ) -// AccessJudgement is a look's judgement of an access's folder, and why, in words. +// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at +// it for a wait, with no path (PlacedID). type AccessJudgement struct { - Kind, Why string + Kind, Why, Placed string } // AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the @@ -662,7 +737,43 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud case v.kind == folderMake: return AccessJudgement{Kind: VerdictMake} case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken: - return AccessJudgement{Kind: VerdictWaitKeep} + return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)} } return AccessJudgement{Kind: VerdictKeep} } + +// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder, +// why, in words with no path. +type Leftover struct { + Module, Access, Name, Path, Why string +} + +// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names +// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each +// is said until its drop walk has run. +func Leftovers() []Leftover { + all, err := KeptFolders.All() + if err != nil { + return nil + } + var out []Leftover + for path, f := range all { + if len(f.IDs) == 0 { + continue + } + module, name := accessName(f.IDs[0]) + dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting" + switch { + case dropping: + out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " + + "0325 §8)"}) + case len(f.Left) > 0: + out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + name + " on a filesystem no longer at its folder could not be removed: they are removed once that " + + "filesystem is back (novox/hq ADR 0325 §8)"}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access }) + return out +} diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 5faeb71..f19c20e 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -61,6 +61,12 @@ func ReconcileKept(ctx context.Context) []string { if ctx.Err() != nil { break } + if len(f.Left) > 0 { + said = append(said, dropLeft(path, f)...) + if f, _ = keptRecord(path); f.Path == "" { + continue + } + } if !f.Kept || sameUsers(f.Users, f.Want) { continue } @@ -69,6 +75,69 @@ func ReconcileKept(ctx context.Context) []string { return said } +// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds +// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they +// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and +// said (Leftovers). +func dropLeft(path string, f store.KeptFolder) []string { + lock := folderLock(path) + lock.Lock() + defer lock.Unlock() + holding := "" + if _, err := lstatWithin(path); err == nil { + holding = mountHolding(path, MountPoints()) + } + var said []string + var still []store.KeptLeft + var mine []store.KeptUser + for _, l := range f.Left { + switch { + case f.Kept && l.Mount == f.Mount: + mine = append(mine, l.Users...) + case l.Mount != holding: + still = append(still, l) + default: + drop := map[uint32]bool{} + for _, u := range l.Users { + drop[u.UID] = true + } + w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop)) + if err != nil || w.Failed != "" { + still = append(still, l) + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed))) + continue + } + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files)) + } + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Left = still + for _, u := range mine { + if _, has := usersOf(k.Users)[u.UID]; !has { + k.Users = append(k.Users, u) + } + } + return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0) + }) + return said +} + +func pathlessOr(err error) string { + if err == nil { + return "" + } + return pathless(err) +} + +func firstOf(a, b string) string { + if a != "" { + return a + } + return b +} + func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string { lock := folderLock(path) lock.Lock() @@ -196,10 +265,15 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } started := time.Now().UTC() _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - if !f.Kept || v.moved { + if v.moved { + *f = movedRecord(*f, v.mount) + } + if !f.Kept { f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil } - f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users + f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids + // The warrant is for the filesystem it was carried out on (§8). + f.TakenOn = v.mount f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} return true }) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 57a7efa..1dc6b20 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -861,3 +862,191 @@ func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) { t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk") } } + +// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset. +func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) { + t.Helper() + dir := aclFolder(t) + dataset = filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib = filepath.Join(dataset, "movies") + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + users...)); err != nil { + t.Fatal(err) + } + return dataset, lib +} + +// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount +// point is at the path: its writes would land on the filesystem below (§9). +func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) { + _, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + MountPoints = func() []string { return []string{"/"} } + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + if len(args) > 0 && args[0] == "container" { + return "", errors.New("no such container") + } + return "", nil + } + d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+ + `,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`) + report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) + if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" { + t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o) + } + for _, r := range ran { + if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") { + t.Fatalf("the container was started: %s", r) + } + } + if !WaitingOnAccess(d)["radarr.app"] { + t.Error("liveness is not told the container waits on its access") + } +} + +// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until +// the drop walk can run; then it is forgotten. +func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { + dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}` + if _, _, err := applyAccesses(t, store.State{}, other); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + left := Leftovers() + if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") || + strings.Contains(left[0].Why, dataset) { + t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left) + } + MountPoints = func() []string { return []string{"/", dataset} } + ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("said after the drop ran: %+v", left) + } + if _, had := keptRecord(lib); had { + t.Error("a folder nobody uses was not forgotten once its entries were out") + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { + t.Error("the entries were not taken out") + } +} + +// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that +// filesystem holds the folder again, they are taken out. +func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") { + t.Fatalf("entries left on the filesystem below are not said: %+v", left) + } + // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { + t.Fatal("the entries left on the filesystem that came back were not taken out") + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("still said after they were taken out: %+v", left) + } +} + +// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on; +// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked. +func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + before := AccessVerdict(context.Background(), lib, true, true) + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o) + } + // And on the next apply too, its record now on the new filesystem. + report, _, _ = applyAccesses(t, store.State{}, taken) + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("the second apply kept it on the old warrant: %+v", o) + } + v := AccessVerdict(context.Background(), lib, true, true) + if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") { + t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before) + } + ReconcileKept(context.Background()) + if got := effectiveOn(t, film, radarr); got != 0 { + t.Fatal("the data on the new filesystem was walked") + } +} + +// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile. +func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) { + was := statBound + statBound = 20 * time.Millisecond + defer func() { statBound = was }() + release := make(chan struct{}) + defer close(release) + var started int32 + var mu sync.Mutex + hang := func(string) (os.FileInfo, error) { + mu.Lock() + started++ + mu.Unlock() + <-release + return nil, nil + } + for i := 0; i < 5; i++ { + if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) { + t.Fatalf("a hung read answered: %v", err) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one path were started while the first hung", started) + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 88083ca..ab77a04 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -309,6 +309,11 @@ const KindDirectory = "directory" // it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12). const KindAccess = "access" +// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet +// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at +// a folder. Unhealthy, said under the access that last named it, until the drop walk has run. +const KindKeptEntries = "kept-entries" + // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" @@ -445,6 +450,9 @@ type AccessWait struct { Access string `json:"access"` Part string `json:"part"` What string `json:"what"` + // Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant + // given for another filesystem (the folder moved) does not answer it, and the controller asks again. + Placed string `json:"placed,omitempty"` } // KeptEntry is one user of a kept folder as its listing says it. diff --git a/internal/store/kept.go b/internal/store/kept.go index ed63fbc..8965692 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -59,6 +59,14 @@ type KeptFolder struct { // declared. Where they differ a walk adds or removes exactly the difference. Users []KeptUser `json:"users,omitempty"` Want []KeptUser `json:"want,omitempty"` + // IDs are the accesses that last named the folder: what a leftover is said under once none does. + IDs []string `json:"ids,omitempty"` + // TakenOn is the mount point the operator's warrant was carried out on (ADR 0325 §8): the controller's word + // that the folder was taken holds for that filesystem alone, and a folder moved to another asks again. + TakenOn string `json:"taken_on,omitempty"` + // Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or + // one gone): taken out by a drop walk once that filesystem holds the folder again, said until then. + Left []KeptLeft `json:"left,omitempty"` // Act is the last walk under the folder: a take, an add or a drop. Act *KeptAct `json:"act,omitempty"` // Sweep is how far the sweep got (ADR 0325 §10). @@ -69,6 +77,12 @@ type KeptFolder struct { OutOfWatches bool `json:"out_of_watches,omitempty"` } +// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are. +type KeptLeft struct { + Mount string `json:"mount"` + Users []KeptUser `json:"users"` +} + // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. type KeptAct struct { // Act is take, add or drop.