From 2026ef36f383b7cfe1168c523057987e5e239a36 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:55:47 +0200 Subject: [PATCH] Judge a kept folder before every walk, and keep nothing a disk mounted over (hq ADR 0325 review) A walk run while a dataset was unmounted wrote onto root's empty mount point, or "dropped" entries from it and forgot the ones on the data; a folder kept while empty below an unnamed mount was then swept whole once the disk mounted, with no warrant. Every walk is now judged as an apply is, a changed mount judges the folder afresh, and systemd .mount units and mounts seen are read for the first-sight rule. Widening a mask caps the entries it limited, so the owning group gains nothing. The watch lets go of removed folders, re-arms on lost notice, reads users under its lock and ends with its context. A drop that cannot read an ACL is counted, a broken folder's words carry no path, a grant's group that could not be left is tried again, and a folder's stat is bounded. --- cmd/mesh-host/kept.go | 5 +- internal/apply/acl.go | 16 +- internal/apply/apply.go | 9 +- internal/apply/identities_test.go | 36 ++++ internal/apply/keep.go | 149 ++++++++++++---- internal/apply/keeper_linux.go | 30 +++- internal/apply/kept_test.go | 280 +++++++++++++++++++++++++++++- internal/apply/placed.go | 82 +++++++-- internal/apply/walk.go | 5 +- internal/apply/walk_linux.go | 10 +- internal/apply/watch_linux.go | 94 ++++++++-- 11 files changed, 641 insertions(+), 75 deletions(-) diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index 0a4e36e..ba742b5 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -162,7 +162,8 @@ func judgeAccess(a judgedAccess) verdict { case apply.VerdictRoot: return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)} case apply.VerdictBroken: - return verdict{state: link.StateUnhealthy, reason: strings.ReplaceAll(v.Why, a.path, "the folder")} + // Its words carry no path (AccessVerdict says them so): the reason reaches the operator's channel. + return verdict{state: link.StateUnhealthy, reason: v.Why} case apply.VerdictMake: return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " + "its data lives (novox/hq ADR 0325 §9)"} @@ -183,7 +184,7 @@ func judgeAccess(a judgedAccess) verdict { look := apply.LookKept(a.path, a.users, apply.SweepLimits) var faults []string if look.Failed != "" { - faults = append(faults, "the sweep could not look: "+strings.ReplaceAll(look.Failed, a.path, "the folder")) + faults = append(faults, "the sweep could not look: "+apply.Pathless(errors.New(look.Failed))) } if len(look.Short) > 0 { faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+ diff --git a/internal/apply/acl.go b/internal/apply/acl.go index 8874ec4..6fbab09 100644 --- a/internal/apply/acl.go +++ b/internal/apply/acl.go @@ -142,7 +142,7 @@ func (a acl) check() error { // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps -// what it gave every other entry; with none yet (a file the mode alone described) the mask starts at the owning +// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. // Nothing else is changed. func keptACL(a acl, need map[uint32]uint16) acl { @@ -157,7 +157,21 @@ func keptACL(a acl, need map[uint32]uint16) acl { } } if i := out.find(aclMask, 0); i >= 0 { + old := out[i].Perm out[i].Perm |= widen + if out[i].Perm != old { + // **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does + // not set here were limited by the old mask: a file made at 0644 under a default ACL carries the + // owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its + // effective right stays exactly as it was. + for j := range out { + e := &out[j] + _, ours := need[e.ID] + if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) { + e.Perm &= old + } + } + } return out } if len(need) == 0 { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index c5f6028..31e4aaa 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -353,12 +353,19 @@ func ApplyMindingWindows( // it (novox/hq ADR 0325 §8), never by removing the access's record. action, detail = "forgotten", "an operator-owned folder is never the host's to remove; the entries "+ "the mesh added go once no access names it" - if gave, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed); said != "" { + gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed) + if said != "" { detail = joinDetail(detail, said) if gave { action = "restored" } } + if len(left) > 0 { + // Kept recorded with what is still to take back, and tried again (ADR 0177's waiting removal). + orphan.Groups = left + known.Record(orphan) + err = fmt.Errorf("%w: %s", errRemovalWaits, said) + } default: action, detail, err = remove(ctx, sys, orphan, run, made) } diff --git a/internal/apply/identities_test.go b/internal/apply/identities_test.go index a31dd09..023e088 100644 --- a/internal/apply/identities_test.go +++ b/internal/apply/identities_test.go @@ -249,3 +249,39 @@ func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) { t.Errorf("the record still holds the grant's groups: %+v", rec) } } + +// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again. +func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) { + dir := t.TempDir() + refuse := true + groups := "nox media" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch name { + case "id": + return groups + "\n", nil + case "gpasswd": + if refuse { + return "", errors.New("gpasswd exited 1: locked") + } + groups = "nox" + } + return "", nil + } + known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir, + Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}} + d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`) + _, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 { + t.Fatalf("a group that could not be left was forgotten: %+v", rec) + } + refuse = false + if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" { + t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups) + } +} diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 000dc1e..440f5eb 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -198,21 +199,23 @@ type folderVerdict struct { // why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link). why string err error + // moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh. + moved bool } -// assessFolder judges a folder as an apply and a look both do (ADR 0325 §7, §9, §12): its disk first, then whose -// it is, then whether the mesh may keep it. +// assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then +// whose it is, then whether the mesh may keep it. Its words carry no path: they reach the operator's channel. func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict { others := recordedMounts(path) - info, err := os.Lstat(path) + info, err := lstatWithin(path) if errors.Is(err, os.ErrNotExist) { m, err := NearestAncestor(path) if err != nil { return folderVerdict{err: err} } if !m.Info.IsDir() { - return folderVerdict{err: fmt.Errorf("%s is not there, and its nearest existing parent %s is not a "+ - "folder (a file or a link); nothing is made under it", path, m.Ancestor)} + return folderVerdict{err: errors.New("the folder is not there, and its nearest existing parent is not " + + "a folder (a file or a link); nothing is made under it")} } pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others) if pl.Away { @@ -220,19 +223,20 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } uid, gid, ok := statOwner(m.Info) if !ok { - return folderVerdict{err: fmt.Errorf("the owner of %s cannot be read on this machine", m.Ancestor)} + return folderVerdict{err: errors.New("the owner of the folder's nearest existing parent cannot be read " + + "on this machine")} } if why := rootOwned(uint32(uid), uint32(gid)); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " + "existing parent is " + why} } - return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade} + return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade, moved: pl.Moved} } if err != nil { - return folderVerdict{err: err} + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} } if !info.IsDir() { - return folderVerdict{err: fmt.Errorf("%s is not a directory, and an access is a shared directory", path)} + return folderVerdict{err: errors.New("the folder is not a directory, and an access is a shared directory")} } pl := placedOn(ctx, path, path, rec.Mount, others) if pl.Away { @@ -240,20 +244,66 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } facts, err := readFolder(path) if err != nil { - return folderVerdict{err: err} + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} } if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } switch { - case rec.Kept: + case rec.Kept && !pl.Moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} case taken: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved} case facts.Empty: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved} } - return folderVerdict{kind: folderWaitKeep, mount: pl.Mount} + // What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty + // below it included (a move): it waits for the operator's word. + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved} +} + +// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose +// server is down blocks a stat for ever, and one access must never hold the apply. +var statBound = 5 * time.Second + +// errNoAnswer is a folder whose read did not return within statBound. +var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)") + +// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own. +func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) } +func statWithin(path string) (os.FileInfo, error) { return within(os.Stat, path) } + +func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + type read struct { + info os.FileInfo + err error + } + done := make(chan read, 1) + go func() { + info, err := stat(path) + done <- read{info, err} + }() + select { + case r := <-done: + return r.info, r.err + case <-time.After(statBound): + return nil, errNoAnswer + } +} + +// absPath is an absolute path in words: what pathless takes out. +var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`) + +// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other +// path said as "a path". +func Pathless(err error) string { return pathless(err) } + +func pathless(err error) string { + var pe *os.PathError + if errors.As(err, &pe) { + return pe.Op + ": " + pe.Err.Error() + } + return absPath.ReplaceAllString(err.Error(), "${1}a path") } // recordedMounts is every mount point recorded for another kept folder: where a disk was once seen. @@ -283,8 +333,11 @@ func RootWords(why string) string { func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, in inputs) (Outcome, error) { // What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5). - gave, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) + gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) out, err := applyAccessFolder(ctx, r, in) + if len(left) > 0 { + out.user, out.groups = previous.User, left + } if said != "" { out.Detail = joinDetail(out.Detail, said) if gave && out.Action == "unchanged" { @@ -307,9 +360,20 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O if v.err != nil { return out, v.err } + if v.moved { + // What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty + // below it): judged afresh, the record of what was kept there no longer holds. + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + *f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act} + return had + }) + } note := "" + if v.moved { + note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh" + } if len(r.Grant) > 0 { - note = "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" + note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" } switch v.kind { case folderWaitDisk: @@ -410,16 +474,41 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O // takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes // back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group. -// Whether any was taken back, and what was said; nothing for an access that granted nothing. +// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next +// apply tries again — and what was said; nothing for an access that granted nothing. func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted, - handed Handed) (bool, string) { + handed Handed) (bool, []string, string) { if previous.User == "" || len(previous.Groups) == 0 { - return false, "" + return false, nil, "" } - account := previous - account.Target = previous.User - gave, said := giveGroupsBack(ctx, sys, account, run, wanted, handed) - return gave, "the group grant ADR 0312 made is taken back: " + previous.User + " " + said + account := previous.User + in, err := system.GroupsOf(ctx, system.Runner(run), account) + if err != nil { + return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+ + "back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err) + } + already := setOf(in) + gave := false + var left, said []string + for _, g := range previous.Groups { + switch { + case !already[g]: + continue + case len(wanted.othersAsk(account, g, previous.ID)) > 0: + said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g, + strings.Join(wanted.othersAsk(account, g, previous.ID), ", "))) + handed.hand(account, g) + continue + } + gone, why := leaveGroup(ctx, sys, account, g, run) + if !gone { + left = append(left, g) + why += "; tried again on the next apply" + } + gave = gave || gone + said = append(said, why) + } + return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ") } // usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)". @@ -469,23 +558,23 @@ func NearestAncestor(path string) (MissingPath, error) { if err := keptPath(path); err != nil { return MissingPath{}, err } - if _, err := os.Lstat(path); err == nil || !errors.Is(err, os.ErrNotExist) { - return MissingPath{}, fmt.Errorf("%s is there, or cannot be read: %v", path, err) + if _, err := lstatWithin(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return MissingPath{}, errors.New("the folder is there, or cannot be read") } var levels []string for p := path; p != "/"; p = filepath.Dir(p) { levels = append([]string{filepath.Base(p)}, levels...) parent := filepath.Dir(p) - info, err := os.Lstat(parent) + info, err := lstatWithin(parent) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { - return MissingPath{}, err + return MissingPath{}, errors.New("a parent of the folder cannot be read: " + pathless(err)) } return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil } - return MissingPath{}, errors.New("no part of the path is there") + return MissingPath{}, errors.New("no part of the folder's path is there") } // accessPaths is every declared access's folder with its id: what no container mounts while it is not there. @@ -511,7 +600,7 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string if !atOrBelow(filepath.Clean(src), path) { continue } - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) { return id } } @@ -563,7 +652,7 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud v := assessFolder(ctx, path, rec, taken) switch { case v.err != nil: - return AccessJudgement{Kind: VerdictBroken, Why: v.err.Error()} + return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)} case v.kind == folderWaitDisk: return AccessJudgement{Kind: VerdictWaitDisk} case !used: diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index e87498a..5faeb71 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -64,15 +64,36 @@ func ReconcileKept(ctx context.Context) []string { if !f.Kept || sameUsers(f.Users, f.Want) { continue } - said = append(said, reconcileOne(path, f)...) + said = append(said, reconcileOne(ctx, path, f)...) } return said } -func reconcileOne(path string, f store.KeptFolder) []string { +func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string { lock := folderLock(path) lock.Lock() defer lock.Unlock() + // **Judged as an apply judges it, before every walk** (§9, §12): a disk not there leaves its mount point — + // an empty folder of root's on the filesystem below — at the path, and a walk there would write onto root's + // folder, or "drop" entries from a folder that never had them and forget the ones on the data. Nothing is + // walked then, and the record stays for the walk that can reach the data. + if v := assessFolder(ctx, path, f, false); v.err != nil || v.kind != folderKeep || v.moved { + why := "the disk that holds it is not there" + switch { + case v.err != nil: + why = pathless(v.err) + case v.kind == folderRoot: + why = RootWords(v.why) + case v.moved || v.kind == folderWaitKeep: + why = "what is at it now is not what the mesh kept there" + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Act = &store.KeptAct{Act: "walk", State: "waiting", Started: time.Now().UTC(), + Why: "not walked: " + why + "; walked once it is"} + return had + }) + return []string{fmt.Sprintf("the kept folder %s: not walked, %s", path, why)} + } have, want := usersOf(f.Users), usersOf(f.Want) drop := map[uint32]bool{} adds := false @@ -98,7 +119,8 @@ func reconcileOne(path string, f store.KeptFolder) []string { rec := actRecord(act, "", "", started, w, err) _ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool { k.Act = rec - if err == nil { + // Done only when every entry was reached and changed: one that failed is walked again. + if err == nil && w.Failed == "" { k.Users = f.Want } return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0) @@ -174,7 +196,7 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } started := time.Now().UTC() _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - if !f.Kept { + if !f.Kept || v.moved { f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil } f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 10f97dd..57a7efa 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -56,9 +56,13 @@ func keeping(t *testing.T, points []string, targets []string) { t.Helper() was, wasPoints, wasTargets := KeptFolders, MountPoints, MountTargets KeptFolders = NewKeptMemory() + forgetMountHistory() MountPoints = func() []string { return append([]string{"/"}, points...) } MountTargets = func(context.Context) []string { return targets } - t.Cleanup(func() { KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets }) + t.Cleanup(func() { + KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets + forgetMountHistory() + }) } // accessJSON is an access of module to path with users, each "identity:uid:use:from". @@ -227,8 +231,9 @@ func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { t.Skipf("no inotify here: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go w.Run(ctx) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + defer func() { cancel(); <-ended }() w.Set(map[string][]store.KeptUser{lib: rec.Want}) narrow := filepath.Join(lib, "narrow") @@ -587,3 +592,272 @@ func TestAnACLReadsBackAsWritten(t *testing.T) { t.Errorf("mask %s, and the entries need rw- and the group had r--", permWords(m)) } } + +func forgetMountHistory() { + mountHistory.Lock() + mountHistory.seen = map[string]bool{} + mountHistory.Unlock() +} + +// Review H1: a walk is judged as an apply is, first. With the dataset not mounted its mount point is what is at the +// path, and neither an add nor a drop walk touches it; the record stays for the walk that can reach the data. +func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) { + dir := aclFolder(t) + dataset := filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib := filepath.Join(dataset, "movies") + both := []string{"mesh-radarr:61001:read-write:container", "mesh-plex:61002:read:container"} + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both...)); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); rec.Mount != dataset || !sameUsers(rec.Users, rec.Want) { + t.Fatalf("not kept on its dataset: %+v", rec) + } + // plex drops it, and the dataset is unmounted before the walk. + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both[0])); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + rec, had := keptRecord(lib) + if !had || len(rec.Users) != 2 { + t.Fatalf("a drop that could not reach the data forgot whose entries are on it: %+v", rec) + } + if rec.Act == nil || rec.Act.State != "waiting" { + t.Errorf("the walk not run is not said: %+v", rec.Act) + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); !still { + t.Fatal("a walk ran where the disk is not there") + } + // The disk back: the drop runs. + MountPoints = func() []string { return []string{"/", dataset} } + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); still { + t.Fatal("the drop did not run once the disk was there") + } +} + +// Review H2: a folder kept while it was empty on the filesystem below a mount nobody named is judged afresh when +// the disk mounts over it: holding data, it waits for the operator's word, and nothing is added to it. +func TestADiskMountedOverAFolderKeptEmptyIsNotKeptWithoutAWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); !rec.Kept || rec.How != store.KeptEmpty || rec.Mount != "/" { + t.Fatalf("not kept as empty on /: %+v", rec) + } + // The pool is imported: a library is at the path now, on its own filesystem. + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + // The new disk's data carries no entries of the mesh's: the test's stand-in made it under the folder kept + // below, so what it inherited there is taken off. + if err := unix.Removexattr(film, aclAccessXattr); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/", pool} } + if v := AccessVerdict(context.Background(), lib, false, true); v.Kind != VerdictWaitKeep { + t.Fatalf("the look judges it %s, and the data on the new disk was never taken", v.Kind) + } + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder whose disk mounted over it was kept without a warrant: %+v", o) + } + if rec, _ := keptRecord(lib); rec.Kept || rec.Mount != pool { + t.Errorf("the record still says kept, or not on the new disk: %+v", rec) + } + if got := effectiveOn(t, film, radarr); got != 0 { + t.Error("an entry was added to the data on the new disk") + } +} + +// Review M3: widening a mask widens nobody else — a file made at 0644 under a default ACL carries the owning group's +// rwx under a mask of r--, and after the mesh widens the mask the owning group still reads only. +func TestWideningTheMaskGivesTheOwningGroupNothingMore(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "shows") + if err := os.Mkdir(lib, 0o775); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lib, 0o775); err != nil { + t.Fatal(err) + } + a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + ep := filepath.Join(lib, "episode") + f, err := os.OpenFile(ep, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.Close() + groupRight := func() uint16 { + acl, _, _ := readACLs(t, ep) + g, _ := acl.perm(aclGroupObj, 0) + m, _ := acl.perm(aclMask, 0) + return g & m + } + before := groupRight() + rec, _ := keptRecord(lib) + LookKept(lib, rec.Want, SweepLimits) + if got := effectiveOn(t, ep, radarr); got&6 != 6 { + t.Fatalf("the sweep did not widen the mask: %s", permWords(got)) + } + if after := groupRight(); after != before { + t.Fatalf("the owning group may %s after the mask widened, and %s before", permWords(after), permWords(before)) + } +} + +// watchedFolder is a kept folder with a watch running on it, its users, and the end of the watch's run. +func watchedFolder(t *testing.T) (string, []store.KeptUser, *Watcher, context.CancelFunc, <-chan struct{}) { + t.Helper() + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "music") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false, + "mesh-lidarr:61001:read-write:container")); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + w, err := NewWatcher() + if err != nil { + t.Skipf("no inotify here: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + // The watch ends before the test's record of kept folders is put back. + t.Cleanup(func() { cancel(); <-ended }) + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + return lib, rec.Want, w, cancel, ended +} + +// Review L4: a watch whose folder goes is let go (IN_IGNORED). +func TestTheWatchOfAFolderRemovedIsLetGo(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + album := filepath.Join(lib, "album") + if err := os.Mkdir(album, 0o755); err != nil { + t.Fatal(err) + } + waitFor(t, "the folder made is watched", func() bool { return w.watched() == 2 }) + if err := os.Remove(album); err != nil { + t.Fatal(err) + } + waitFor(t, "the watch of a folder removed is let go", func() bool { return w.watched() == 1 }) +} + +// Review L4: notice lost re-arms every watch, and the record says the watch is whole again. +func TestNoticeLostReArmsTheWatch(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + w.lost() + if w.watched() != 1 { + t.Fatalf("notice lost did not re-arm the watch: %d watched", w.watched()) + } + if r, _ := keptRecord(lib); r.OutOfWatches { + t.Error("a re-armed watch is still said out of watches") + } + late := filepath.Join(lib, "late") + f, _ := os.OpenFile(late, os.O_CREATE|os.O_WRONLY, 0o644) + f.Close() + waitFor(t, "the re-armed watch repairs", func() bool { return effectiveOn(t, late, radarr)&6 == 6 }) +} + +// Review L4: users given while notices are heard are read under the lock (run with -race). +func TestUsersGivenWhileNoticesAreHeardAreReadUnderTheLock(t *testing.T) { + lib, users, w, _, _ := watchedFolder(t) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + w.Set(map[string][]store.KeptUser{lib: users}) + } + } + }() + for i := 0; i < 50; i++ { + _ = os.WriteFile(filepath.Join(lib, fmt.Sprintf("f%d", i)), nil, 0o644) + } + waitFor(t, "the last file is repaired", func() bool { + return effectiveOn(t, filepath.Join(lib, "f49"), radarr)&6 == 6 + }) + close(stop) + <-done +} + +// Review L4: the watch ends with its context, however quiet the machine. +func TestTheWatchEndsWithItsContext(t *testing.T) { + _, _, _, cancel, ended := watchedFolder(t) + cancel() + select { + case <-ended: + case <-time.After(2 * time.Second): + t.Fatal("the watch did not end with its context") + } +} + +// Review L6: a broken folder's words carry no path. +func TestABrokenFoldersWordsCarryNoPath(t *testing.T) { + dir := t.TempDir() + keeping(t, nil, nil) + file := filepath.Join(dir, "notafolder") + if err := os.WriteFile(file, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, path := range []string{file, filepath.Join(file, "below")} { + v := AccessVerdict(context.Background(), path, false, true) + if v.Kind != VerdictBroken || strings.Contains(v.Why, "/") { + t.Errorf("%s: %+v", path, v) + } + } +} + +// Review H2: a systemd .mount unit says where a disk is meant to be, as fstab does; and a mount point seen mounted +// once is remembered, so a folder below it is a missing disk on first sight when it is gone. +func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) { + units := t.TempDir() + unit := "[Unit]\nDescription=pool\n\n[Mount]\nWhat=/dev/disk/by-label/pool\nWhere=/srv/pool\nType=ext4\n" + if err := os.WriteFile(filepath.Join(units, "srv-pool.mount"), []byte(unit), 0o644); err != nil { + t.Fatal(err) + } + if got := mountUnitTargets([]string{units}); len(got) != 1 || got[0] != "/srv/pool" { + t.Fatalf("read %v from a .mount unit", got) + } + + dir := t.TempDir() + disk := filepath.Join(dir, "disk") + if err := os.Mkdir(disk, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{disk}, nil) + _ = placedOn(context.Background(), disk, disk, "", nil) // seen mounted once + MountPoints = func() []string { return []string{"/"} } + if pl := placedOn(context.Background(), filepath.Join(disk, "films"), disk, "", nil); !pl.Away { + t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk") + } +} diff --git a/internal/apply/placed.go b/internal/apply/placed.go index d96163b..7637e96 100644 --- a/internal/apply/placed.go +++ b/internal/apply/placed.go @@ -17,20 +17,49 @@ import ( // holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the // kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged // against that record: held by another mount point than the one recorded while the recorded one is not mounted is -// the disk not there. A recorded mount point that is mounted is the disk there, whatever now holds the folder (a -// mount made deeper is a placement, not a missing disk), and the record follows it. +// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is +// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made +// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's +// word as any folder holding data does (ADR 0325 §8). // // **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be: -// every mount target in /etc/fstab, every mount point a ZFS dataset names (`zfs list`, when the machine has zfs), -// and every mount point recorded for another kept folder. A target that is the folder or above it, lies below the -// mount point that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that -// holds what is there is the placed one. A mount nothing names in those places (a hand `mount` of a disk that is not -// in fstab, then unmounted before the engine ever saw the folder) is not known to be missing: said here, because the -// folder would then be made on the filesystem below it. +// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names +// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every +// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point +// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what +// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list` +// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen +// empty below it is kept as empty, and the move above takes it back the moment the disk is there. // MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test // can stand in for mounts it cannot make. var MountPoints = func() []string { + return readMountPoints() +} + +// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history"). +var mountHistory = struct { + sync.Mutex + seen map[string]bool +}{seen: map[string]bool{}} + +// sawMounted adds points to the history, and is the history: every mount point seen, never "/". +func sawMounted(points []string) []string { + mountHistory.Lock() + defer mountHistory.Unlock() + for _, p := range points { + if p != "/" { + mountHistory.seen[p] = true + } + } + out := make([]string, 0, len(mountHistory.seen)) + for p := range mountHistory.seen { + out = append(out, p) + } + return out +} + +func readMountPoints() []string { raw, err := os.ReadFile("/proc/self/mountinfo") if err != nil { return []string{"/"} @@ -60,6 +89,7 @@ func readMountTargets(ctx context.Context) []string { if raw, err := os.ReadFile("/etc/fstab"); err == nil { out = append(out, fstabTargets(string(raw))...) } + out = append(out, mountUnitTargets(systemdUnitDirs)...) if _, err := exec.LookPath("zfs"); err == nil { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -74,6 +104,33 @@ func readMountTargets(ctx context.Context) []string { return out } +// systemdUnitDirs are where systemd keeps the units a machine mounts by. +var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator", + "/usr/lib/systemd/system", "/lib/systemd/system"} + +// mountUnitTargets reads the `Where=` of every .mount unit in dirs. +func mountUnitTargets(dirs []string) []string { + var out []string + for _, dir := range dirs { + names, err := filepath.Glob(filepath.Join(dir, "*.mount")) + if err != nil { + continue + } + for _, name := range names { + raw, err := os.ReadFile(name) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") { + out = append(out, filepath.Clean(strings.TrimSpace(v))) + } + } + } + } + return out +} + // mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes // undone. func mountinfoPoints(raw string) []string { @@ -172,6 +229,9 @@ type placement struct { Mount string // Away is that the disk the folder was placed on is not there. Away bool + // Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what + // was kept. + Moved bool } // placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent), @@ -183,10 +243,12 @@ func placedOn(ctx context.Context, path, existing, recorded string, others []str mounted[p] = true } have := mountHolding(existing, points) + seen := sawMounted(points) if recorded != "" { - return placement{Mount: have, Away: have != recorded && !mounted[recorded]} + return placement{Mount: have, Away: have != recorded && !mounted[recorded], + Moved: have != recorded && mounted[recorded]} } - for _, t := range append(MountTargets(ctx), others...) { + for _, t := range append(append(MountTargets(ctx), others...), seen...) { if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] { return placement{Mount: have, Away: true} } diff --git a/internal/apply/walk.go b/internal/apply/walk.go index a48ece9..51d9d58 100644 --- a/internal/apply/walk.go +++ b/internal/apply/walk.go @@ -2,7 +2,6 @@ package apply import ( "errors" - "fmt" "path/filepath" "time" ) @@ -60,8 +59,8 @@ var ErrNoACL = errors.New("the filesystem allows no access entries") // never name it. func keptPath(path string) error { if !filepath.IsAbs(path) || filepath.Clean(path) != path { - return fmt.Errorf("%q is not an access's folder as the engine states it (absolute, no `..`, no doubled or "+ - "trailing separator); nothing was changed", path) + return errors.New("the folder is not stated as the engine states one (absolute, no `..`, no doubled or " + + "trailing separator); nothing was changed") } return nil } diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go index d307ff3..1f334dc 100644 --- a/internal/apply/walk_linux.go +++ b/internal/apply/walk_linux.go @@ -62,7 +62,7 @@ func openKept(path string, limits WalkLimits, stats walkStats, visit visitFunc) return nil, -1, st, errors.New("the folder is not there; nothing was changed") } if err != nil { - return nil, -1, st, fmt.Errorf("the folder is not walked: %v", err) + return nil, -1, st, errors.New("the folder is not walked: " + pathless(err)) } if mw.At != path { unix.Close(fd) @@ -363,11 +363,15 @@ func dropVisit(drop map[uint32]bool) visitFunc { continue } a, err := getACL(fd, name, mode) - if errors.Is(err, ErrNoACL) || a == nil { + if errors.Is(err, ErrNoACL) { continue } if err != nil { - return changed, false, false, err + // Counted unrepaired and said by the walk, which then walks it again: never read as "nothing to drop". + return changed, false, true, err + } + if a == nil { + continue } if left := withoutUsers(a, drop); len(left) != len(a) { if err := setACL(fd, name, left); err != nil { diff --git a/internal/apply/watch_linux.go b/internal/apply/watch_linux.go index e79ce16..8bbb43f 100644 --- a/internal/apply/watch_linux.go +++ b/internal/apply/watch_linux.go @@ -57,7 +57,8 @@ type watchedRoot struct { // NewWatcher opens the machine's notice of changes. func NewWatcher() (*Watcher, error) { - fd, err := unix.InotifyInit1(unix.IN_CLOEXEC) + // Not blocking: Run polls it, so the end of its context ends it whatever the kernel has to say. + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) if err != nil { return nil, err } @@ -149,21 +150,29 @@ func (w *Watcher) watchBelow(path string, root *watchedRoot, fd int, rel []strin } } -// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder. +// watchPoll is how long Run waits for notice before it looks whether its context ended. +const watchPoll = 250 * time.Millisecond + +// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder, and then closes the +// watch: polled, so a quiet machine's watch ends as soon as a busy one's. func (w *Watcher) Run(ctx context.Context) { - go func() { - <-ctx.Done() - unix.Close(w.fd) - }() + defer unix.Close(w.fd) buf := make([]byte, 64*1024) - for { - n, err := unix.Read(w.fd, buf) - if err != nil || n <= 0 { - if ctx.Err() != nil || !errors.Is(err, unix.EINTR) { - return - } + for ctx.Err() == nil { + ready, err := unix.Poll([]unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}}, int(watchPoll/time.Millisecond)) + if err != nil && !errors.Is(err, unix.EINTR) { + return + } + if ready <= 0 { continue } + n, err := unix.Read(w.fd, buf) + if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR) { + continue + } + if err != nil || n <= 0 { + return + } for at := 0; at+unix.SizeofInotifyEvent <= n; { ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[at])) nameAt := at + unix.SizeofInotifyEvent @@ -173,36 +182,80 @@ func (w *Watcher) Run(ctx context.Context) { w.lost() continue } + if ev.Mask&unix.IN_IGNORED != 0 { + // The watch is gone — its folder removed, or let go by Set: what pointed at it goes too. + w.forget(int(ev.Wd)) + continue + } w.heard(int(ev.Wd), name, ev.Mask) } } } -// lost is notice lost: every kept folder is said to rely on its sweep until it is watched afresh. +// lost is notice lost (the kernel's queue overflowed): every kept folder is watched afresh, and what was made +// meanwhile is the sweep's. func (w *Watcher) lost() { w.mu.Lock() defer w.mu.Unlock() for path, root := range w.roots { - root.out = true + for _, wd := range root.wds { + _, _ = unix.InotifyRmWatch(w.fd, uint32(wd)) + delete(w.dirs, wd) + } + root.wds, root.out = nil, false + w.watchTree(path, root) w.say(path, root) } } +// forget lets go of a watch the kernel removed. +func (w *Watcher) forget(wd int) { + w.mu.Lock() + defer w.mu.Unlock() + dir, ok := w.dirs[wd] + if !ok { + return + } + delete(w.dirs, wd) + if root := w.roots[dir.root]; root != nil { + for i, had := range root.wds { + if had == wd { + root.wds = append(root.wds[:i], root.wds[i+1:]...) + break + } + } + w.say(dir.root, root) + } +} + +// watched is how many folders are watched now. +func (w *Watcher) watched() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.dirs) +} + // heard repairs one entry a watch heard of: name in the watched folder wd, or the folder itself. func (w *Watcher) heard(wd int, name string, mask uint32) { w.mu.Lock() dir, ok := w.dirs[wd] root := w.roots[dir.root] + var users map[uint32]bool + var dev uint64 + if root != nil { + // Read under the lock: Set gives a root new users while notices are heard. + users, dev = root.users, root.dev + } w.mu.Unlock() if !ok || root == nil { return } - fd, err := openBelow(dir.root, dir.rel, root.dev) + fd, err := openBelow(dir.root, dir.rel, dev) if err != nil { return } defer unix.Close(fd) - visit := keepVisit(root.users) + visit := keepVisit(users) if name == "" { var st unix.Stat_t if unix.Fstat(fd, &st) == nil { @@ -211,7 +264,7 @@ func (w *Watcher) heard(wd int, name string, mask uint32) { return } var st unix.Stat_t - if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != root.dev { + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != dev { return } isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR @@ -235,9 +288,14 @@ func (w *Watcher) heard(wd int, name string, mask uint32) { if isDir && mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 { // A folder made or moved in is watched, and what was moved in with it is the sweep's. w.mu.Lock() + defer w.mu.Unlock() + if w.roots[dir.root] != root { + // Let go by Set meanwhile: nothing more is watched for it. + unix.Close(efd) + return + } w.watchBelow(dir.root, root, efd, append(append([]string(nil), dir.rel...), name)) w.say(dir.root, root) - w.mu.Unlock() return } unix.Close(efd)