Files
mesh-host/internal/apply/keep.go
T
jschoubben 2026ef36f3 Judge a kept folder before every walk, and keep nothing a disk mounted over (hq ADR 0325 review)
A walk run while a dataset was unmounted wrote onto root's empty mount point, or
"dropped" entries from it and forgot the ones on the data; a folder kept while empty
below an unnamed mount was then swept whole once the disk mounted, with no warrant.
Every walk is now judged as an apply is, a changed mount judges the folder afresh, and
systemd .mount units and mounts seen are read for the first-sight rule. Widening a
mask caps the entries it limited, so the owning group gains nothing. The watch lets
go of removed folders, re-arms on lost notice, reads users under its lock and ends
with its context. A drop that cannot read an ACL is counted, a broken folder's words
carry no path, a grant's group that could not be left is tried again, and a folder's
stat is bounded.
2026-10-12 18:55:47 +02:00

669 lines
24 KiB
Go

package apply
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// A shared folder is kept usable through access entries (novox/hq ADR 0325, issue 567).
//
// Several modules use one folder of the operator's — a library one imports into, one renames in, one plays from —
// each as its own identity. The engine keeps, for every placed folder, one ACL entry per identity that uses it,
// on the folder and as its default for everything made under it, and changes nothing else of the path (§5).
//
// **When it may keep a folder.** Only one that is the mesh's to keep: taken on the operator's warrant (§8), or
// holding no data yet — empty when first seen, or made by the engine (§9). A folder that holds data and was never
// taken is stated waiting, with a wait of kind access, until the operator lets the mesh keep it (§7); one whose disk
// is not there waits for the disk (§9); root's data on its placed filesystem is never kept, and is unhealthy (§12).
//
// **What every apply does.** It judges the folder (assessFolder), makes a missing one where its data lives, and
// sets the users' entries on the folder itself. What lies under a taken folder is the walks' (ReconcileKept): a
// user added has its entries added to everything under it, a user dropped has exactly its own removed, outside
// the apply and bounded. The watch and the sweep keep what is made later usable (§6, §10).
// keptRecords is where the engine keeps what it knows of each kept or seen folder: the file beside the state in
// the engine, memory in a test.
type keptRecords interface {
All() (map[string]store.KeptFolder, error)
Update(path string, change func(f *store.KeptFolder, had bool) bool) error
}
// KeptFolders is the engine's record of the folders it keeps. The engine sets the file beside its state.
var KeptFolders keptRecords = NewKeptMemory()
// keptMemory is a keptRecords held in memory.
type keptMemory struct {
mu sync.Mutex
m map[string]store.KeptFolder
}
// NewKeptMemory is an empty record held in memory: a test's, or a one-shot apply's.
func NewKeptMemory() keptRecords { return &keptMemory{m: map[string]store.KeptFolder{}} }
func (k *keptMemory) All() (map[string]store.KeptFolder, error) {
k.mu.Lock()
defer k.mu.Unlock()
out := map[string]store.KeptFolder{}
for p, f := range k.m {
out[p] = f
}
return out, nil
}
func (k *keptMemory) Update(path string, change func(f *store.KeptFolder, had bool) bool) error {
k.mu.Lock()
defer k.mu.Unlock()
f, had := k.m[path]
f.Path = path
if change(&f, had) {
k.m[path] = f
} else {
delete(k.m, path)
}
return nil
}
// keptRecord is the record of path, and whether there is one. A record that cannot be read is none: the folder
// is judged afresh, which waits rather than keeps.
func keptRecord(path string) (store.KeptFolder, bool) {
all, err := KeptFolders.All()
if err != nil {
return store.KeptFolder{}, false
}
f, had := all[path]
return f, had
}
// folderWant is what every access naming one folder asks of it: the accesses, each uid at the widest use any of
// them gives it (ADR 0325 §4: the tool runner's account stands for every module whose tools use it), and whether
// the controller says it was taken.
type folderWant struct {
path string
ids []string
users map[uint32]store.KeptUser
taken bool
}
// foldersOf is every declared access's folder, by its clean path. A folder the guard refuses is not read (issue
// 339).
func foldersOf(resources []declaration.Resource, refused map[string]error) map[string]*folderWant {
out := map[string]*folderWant{}
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok {
continue
}
path := filepath.Clean(a.Path)
if refused[path] != nil {
continue
}
f := out[path]
if f == nil {
f = &folderWant{path: path, users: map[uint32]store.KeptUser{}}
out[path] = f
}
f.ids = append(f.ids, a.ID)
f.taken = f.taken || a.Taken
for _, u := range a.Users {
had, seen := f.users[uint32(u.UID)]
if seen && (had.Use == declaration.AccessReadWrite || u.Use != declaration.AccessReadWrite) {
continue
}
f.users[uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), Use: u.Use, From: u.From}
}
}
return out
}
// list is the users, by uid.
func (f *folderWant) list() []store.KeptUser {
if f == nil {
return nil
}
out := make([]store.KeptUser, 0, len(f.users))
for _, u := range f.users {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].UID < out[j].UID })
return out
}
// usersOf is a list of users as each uid and whether it writes.
func usersOf(us []store.KeptUser) map[uint32]bool {
out := map[uint32]bool{}
for _, u := range us {
out[u.UID] = out[u.UID] || u.Use == declaration.AccessReadWrite
}
return out
}
// sameUsers is whether two lists give the same uids the same use.
func sameUsers(a, b []store.KeptUser) bool {
x, y := usersOf(a), usersOf(b)
if len(x) != len(y) {
return false
}
for uid, rw := range x {
if w, ok := y[uid]; !ok || w != rw {
return false
}
}
return true
}
// accessName is the module and its own id for an access, from the resource id `<module>.<id>`.
func accessName(id string) (module, name string) {
if m, ok := moduleOf(id); ok {
return m, id[len(m)+1:]
}
return "", id
}
// KeepWords and DiskWords are what an access wait says the operator gives (ADR 0325 §7): never a path.
func KeepWords(name string) string { return "let the mesh keep the shared folder " + name }
func DiskWords(name string) string { return "mount the disk that holds " + name }
// The judgements of a folder.
const (
// folderKeep is a folder the mesh keeps: taken, or holding no data yet.
folderKeep = "keep"
// folderMake is a folder absent under a parent on its placed filesystem, not root's: made, then kept.
folderMake = "make"
// folderWaitKeep is a folder that holds data and was never taken: it waits for the operator's word.
folderWaitKeep = "wait-keep"
// folderWaitDisk is a folder whose disk is not there.
folderWaitDisk = "wait-disk"
// folderRoot is root's data on its placed filesystem: never kept.
folderRoot = "root"
)
// folderVerdict is the judgement of one folder, with what it read.
type folderVerdict struct {
kind string
// how is why a folder kept is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptMade.
how string
// mount is the mount point that holds what is there.
mount string
// why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link).
why string
err error
// moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh.
moved bool
}
// assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then
// whose it is, then whether the mesh may keep it. Its words carry no path: they reach the operator's channel.
func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict {
others := recordedMounts(path)
info, err := lstatWithin(path)
if errors.Is(err, os.ErrNotExist) {
m, err := NearestAncestor(path)
if err != nil {
return folderVerdict{err: err}
}
if !m.Info.IsDir() {
return folderVerdict{err: errors.New("the folder is not there, and its nearest existing parent is not " +
"a folder (a file or a link); nothing is made under it")}
}
pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others)
if pl.Away {
return folderVerdict{kind: folderWaitDisk, mount: pl.Mount}
}
uid, gid, ok := statOwner(m.Info)
if !ok {
return folderVerdict{err: errors.New("the owner of the folder's nearest existing parent cannot be read " +
"on this machine")}
}
if why := rootOwned(uint32(uid), uint32(gid)); why != "" {
return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " +
"existing parent is " + why}
}
return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade, moved: pl.Moved}
}
if err != nil {
return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))}
}
if !info.IsDir() {
return folderVerdict{err: errors.New("the folder is not a directory, and an access is a shared directory")}
}
pl := placedOn(ctx, path, path, rec.Mount, others)
if pl.Away {
return folderVerdict{kind: folderWaitDisk, mount: pl.Mount}
}
facts, err := readFolder(path)
if err != nil {
return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))}
}
if why := rootOwned(facts.UID, facts.GID); why != "" {
return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why}
}
switch {
case rec.Kept && !pl.Moved:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How}
case taken:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved}
case facts.Empty:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved}
}
// What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty
// below it included (a move): it waits for the operator's word.
return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved}
}
// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
// server is down blocks a stat for ever, and one access must never hold the apply.
var statBound = 5 * time.Second
// errNoAnswer is a folder whose read did not return within statBound.
var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) }
func statWithin(path string) (os.FileInfo, error) { return within(os.Stat, path) }
func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
type read struct {
info os.FileInfo
err error
}
done := make(chan read, 1)
go func() {
info, err := stat(path)
done <- read{info, err}
}()
select {
case r := <-done:
return r.info, r.err
case <-time.After(statBound):
return nil, errNoAnswer
}
}
// absPath is an absolute path in words: what pathless takes out.
var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`)
// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other
// path said as "a path".
func Pathless(err error) string { return pathless(err) }
func pathless(err error) string {
var pe *os.PathError
if errors.As(err, &pe) {
return pe.Op + ": " + pe.Err.Error()
}
return absPath.ReplaceAllString(err.Error(), "${1}a path")
}
// recordedMounts is every mount point recorded for another kept folder: where a disk was once seen.
func recordedMounts(except string) []string {
all, err := KeptFolders.All()
if err != nil {
return nil
}
var out []string
for p, f := range all {
if p != except && f.Mount != "" && f.Mount != "/" {
out = append(out, f.Mount)
}
}
return out
}
// RootWords is the reason root's data is stated unhealthy with (ADR 0325 §12): never a path.
func RootWords(why string) string {
return why + ", and root's data is never kept: the operator places the data elsewhere or gives it an owner " +
"(novox/hq ADR 0325 §12)"
}
// applyAccess is one access, applied (ADR 0325): its folder judged, made when it is missing where its data lives,
// and its users' entries kept on it when the mesh may keep it; waiting otherwise, and failing for root's data.
// Nothing is changed on the folder but the entries the mesh adds and the mask that shows them.
func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
in inputs) (Outcome, error) {
// What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5).
gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed)
out, err := applyAccessFolder(ctx, r, in)
if len(left) > 0 {
out.user, out.groups = previous.User, left
}
if said != "" {
out.Detail = joinDetail(out.Detail, said)
if gave && out.Action == "unchanged" {
// Updated, so what names the access under restart-on — the tool runner — holds the groups it has now.
out.Action = "updated"
}
}
return out, err
}
// applyAccessFolder is applyAccess's folder.
func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (Outcome, error) {
out := begin(r)
path := filepath.Clean(r.Path)
want := in.folders[path]
users := want.list()
_, name := accessName(r.ID)
rec, _ := keptRecord(path)
v := assessFolder(ctx, path, rec, want != nil && want.taken)
if v.err != nil {
return out, v.err
}
if v.moved {
// What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty
// below it): judged afresh, the record of what was kept there no longer holds.
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
*f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act}
return had
})
}
note := ""
if v.moved {
note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh"
}
if len(r.Grant) > 0 {
note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)"
}
switch v.kind {
case folderWaitDisk:
out.Action = "waiting"
out.Detail = "the disk that holds it is not there: nothing is made, and nothing mounts it until it is " +
"(novox/hq ADR 0325 §9); waits for the operator to " + DiskWords(name) + note
return out, nil
case folderRoot:
if len(users) == 0 {
out.Action, out.Detail = "unchanged", "present or not, nothing is kept: no identity uses it"+note
return out, nil
}
return out, errors.New(RootWords(v.why) + "; nothing was made or changed")
case folderMake:
made, err := CreatePath(path)
if err != nil {
return out, err
}
out.Action = "created"
out.Detail = fmt.Sprintf("made %d level(s) as its nearest existing parent is: owner, group and mode "+
"(novox/hq ADR 0325 §9)", made)
if len(users) == 0 {
out.Detail += note
return out, nil
}
case folderWaitKeep:
if len(users) == 0 {
out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"+note
return out, nil
}
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Mount, f.Want = v.mount, users
return true
})
out.Action = "waiting"
out.Detail = "it holds data the mesh has not taken: nothing is changed on it until the operator's warrant " +
"(novox/hq ADR 0325 §7, §8); waits for the operator to " + KeepWords(name) + note
return out, nil
}
if len(users) == 0 {
// Nobody uses it any more: what the mesh added goes on the next walk.
dropping := false
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
f.Want = nil
dropping = len(f.Users) > 0
return had
})
if out.Action == "" {
out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"
}
if dropping {
out.Detail += "; the entries the mesh added are taken out by the next walk"
}
out.Detail += note
return out, nil
}
changed, err := keepTop(path, usersOf(users))
if errors.Is(err, ErrNoACL) {
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Mount, f.Want = v.mount, users
return true
})
out.Action = "waiting"
out.Detail = "its filesystem allows no access entries here: the take act lets it, on the operator's word " +
"(novox/hq ADR 0325 §8); waits for the operator to " + KeepWords(name) + note
return out, nil
}
if err != nil {
return out, err
}
var pending string
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
first := !f.Kept
f.Kept, f.Mount, f.Want = true, v.mount, users
if first {
f.How, f.At = v.how, time.Now().UTC()
// A folder that held nothing has nothing under it to walk; one the controller says was taken
// without this engine's record is walked whole.
if v.how == store.KeptEmpty || v.how == store.KeptMade {
f.Users = users
}
}
if !sameUsers(f.Users, f.Want) {
pending = "; the entries under it follow on the next walk"
}
return true
})
if out.Action == "" {
out.Action = "unchanged"
if changed {
out.Action = "updated"
}
}
out.Detail = joinDetail(out.Detail, "kept for "+usersWords(users)+": its own entries and default, nothing "+
"else of it changed (novox/hq ADR 0325 §5)"+pending+note)
return out, nil
}
// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes
// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group.
// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next
// apply tries again — and what was said; nothing for an access that granted nothing.
func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted,
handed Handed) (bool, []string, string) {
if previous.User == "" || len(previous.Groups) == 0 {
return false, nil, ""
}
account := previous.User
in, err := system.GroupsOf(ctx, system.Runner(run), account)
if err != nil {
return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+
"back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err)
}
already := setOf(in)
gave := false
var left, said []string
for _, g := range previous.Groups {
switch {
case !already[g]:
continue
case len(wanted.othersAsk(account, g, previous.ID)) > 0:
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g,
strings.Join(wanted.othersAsk(account, g, previous.ID), ", ")))
handed.hand(account, g)
continue
}
gone, why := leaveGroup(ctx, sys, account, g, run)
if !gone {
left = append(left, g)
why += "; tried again on the next apply"
}
gave = gave || gone
said = append(said, why)
}
return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ")
}
// usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)".
func usersWords(users []store.KeptUser) string {
words := make([]string, 0, len(users))
for _, u := range users {
who := u.Identity
if u.From == declaration.FromToolRunner {
who = "the tool runner's " + who
}
words = append(words, who+" ("+u.Use+")")
}
return strings.Join(words, ", ")
}
// forgetUndeclared marks every recorded folder no declared access names any more: nobody wants its entries, and
// the next walk takes out exactly what the mesh added, then forgets it.
func forgetUndeclared(folders map[string]*folderWant) {
all, err := KeptFolders.All()
if err != nil {
return
}
for path := range all {
if folders[path] != nil {
continue
}
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Want = nil
// A folder never kept, or kept with nobody's entries left, is forgotten now.
return f.Kept && len(f.Users) > 0
})
}
}
// MissingPath is an accessed folder that is not there, and its nearest existing ancestor.
type MissingPath struct {
Ancestor string
// Levels are the names to make below Ancestor, outermost first.
Levels []string
// Info is the ancestor as lstat reads it: a link is not a directory.
Info os.FileInfo
}
// NearestAncestor reads, for a path that is not there, its nearest existing ancestor by lstat. An error for a
// path that is there, one not stated exactly, or an ancestor that cannot be read.
func NearestAncestor(path string) (MissingPath, error) {
if err := keptPath(path); err != nil {
return MissingPath{}, err
}
if _, err := lstatWithin(path); err == nil || !errors.Is(err, os.ErrNotExist) {
return MissingPath{}, errors.New("the folder is there, or cannot be read")
}
var levels []string
for p := path; p != "/"; p = filepath.Dir(p) {
levels = append([]string{filepath.Base(p)}, levels...)
parent := filepath.Dir(p)
info, err := lstatWithin(parent)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return MissingPath{}, errors.New("a parent of the folder cannot be read: " + pathless(err))
}
return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil
}
return MissingPath{}, errors.New("no part of the folder's path is there")
}
// accessPaths is every declared access's folder with its id: what no container mounts while it is not there.
func accessPaths(resources []declaration.Resource, refused map[string]error) map[string]string {
out := map[string]string{}
for _, r := range resources {
if a, ok := r.(*declaration.Access); ok && refused[filepath.Clean(a.Path)] == nil {
out[filepath.Clean(a.Path)] = a.ID
}
}
return out
}
// mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there
// now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026).
func mountsMissing(c *declaration.Container, accesses map[string]string) string {
for _, v := range c.Volumes {
src := mountSource(v)
if !strings.HasPrefix(src, "/") {
continue
}
for path, id := range accesses {
if !atOrBelow(filepath.Clean(src), path) {
continue
}
if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) {
return id
}
}
}
return ""
}
// WaitingOnAccess is every container of the declaration that waits for an accessed folder not there, read now:
// not created by the apply, so not judged alive or dead — the access says why.
func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
out := map[string]bool{}
if d == nil {
return out
}
accesses := accessPaths(d.Resources, refusedPlaces(d.Resources))
if len(accesses) == 0 {
return out
}
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" {
out[c.ID] = true
}
}
return out
}
// The judgements of an access as a look says them.
const (
VerdictKeep = "keep"
VerdictMake = "make"
VerdictWaitKeep = "wait-keep"
VerdictWaitDisk = "wait-disk"
VerdictRoot = "root"
VerdictBroken = "broken"
// VerdictNobody is a folder no identity uses: nothing is kept, and only a missing disk is said.
VerdictNobody = "nobody"
)
// AccessJudgement is a look's judgement of an access's folder, and why, in words.
type AccessJudgement struct {
Kind, Why string
}
// AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the
// controller says it was taken, used whether any identity uses it. A folder the mesh could keep that no apply has
// kept — its filesystem allowing no ACLs — waits as one not taken does.
func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJudgement {
rec, _ := keptRecord(path)
v := assessFolder(ctx, path, rec, taken)
switch {
case v.err != nil:
return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)}
case v.kind == folderWaitDisk:
return AccessJudgement{Kind: VerdictWaitDisk}
case !used:
return AccessJudgement{Kind: VerdictNobody}
case v.kind == folderRoot:
return AccessJudgement{Kind: VerdictRoot, Why: v.why}
case v.kind == folderMake:
return AccessJudgement{Kind: VerdictMake}
case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken:
return AccessJudgement{Kind: VerdictWaitKeep}
}
return AccessJudgement{Kind: VerdictKeep}
}