jschoubben
aad33dfe5c
The controller cannot know the operator's account's uid, names an access by its declared id (which holds a slash), and should not parse words. A tool runner's account may now come by name alone and is read here (refused when absent or root's, its uid said in the access's listing); the take act matches the declared id exactly; an access wait says its reason (keep or disk) and, for a filesystem that refuses ACLs, whether the take act can enable them.
474 lines
17 KiB
Go
474 lines
17 KiB
Go
//go:build linux
|
|
|
|
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// The walks under a kept folder, beside the apply (novox/hq ADR 0325 §6, §8, §10).
|
|
//
|
|
// The take act adds the users' entries to everything under a folder on the operator's warrant; ReconcileKept adds a
|
|
// joining user's and removes exactly a dropped user's under a folder already taken; the sweep looks at a bounded
|
|
// part of each kept folder on every look, going on where the last stopped, and puts back what an app narrowed.
|
|
// One walk at a time per folder: a sweep finds a folder being walked and waits for the next look.
|
|
|
|
// folderLocks is one lock per kept folder, held by every walk of it.
|
|
var folderLocks sync.Map
|
|
|
|
func folderLock(path string) *sync.Mutex {
|
|
l, _ := folderLocks.LoadOrStore(path, &sync.Mutex{})
|
|
return l.(*sync.Mutex)
|
|
}
|
|
|
|
// actRecord is a walk's counts as the record keeps them.
|
|
func actRecord(act, warrant, by string, started time.Time, w *aclWalk, err error) *store.KeptAct {
|
|
c := w.counts
|
|
rec := &store.KeptAct{Act: act, Warrant: warrant, By: by, State: "done", Started: started, Ended: time.Now().UTC(),
|
|
Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other, Mounts: c.Mounts}
|
|
switch {
|
|
case errors.Is(err, ErrWalkBound):
|
|
rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on at the next apply"
|
|
case err != nil:
|
|
rec.State, rec.Why = "failed", err.Error()
|
|
case w.Failed != "":
|
|
rec.Why = fmt.Sprintf("%d entries could not be given their entries: %s", c.Unrepaired, w.Failed)
|
|
}
|
|
return rec
|
|
}
|
|
|
|
// ReconcileKept walks every kept folder whose entries under it are not what the last apply declared: a user that
|
|
// joined has its entries added to everything under it, a dropped user — or one whose use changed — has exactly its
|
|
// own taken out (ADR 0325 §8). Bounded by ActLimits; a walk stopped is gone on with at the next call. A folder
|
|
// nobody uses any more is forgotten once its entries are out. What it did, in words, for the engine to say.
|
|
func ReconcileKept(ctx context.Context) []string {
|
|
all, err := KeptFolders.All()
|
|
if err != nil {
|
|
return []string{"the kept folders could not be read: " + err.Error()}
|
|
}
|
|
var said []string
|
|
for path, f := range all {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
if len(f.Left) > 0 {
|
|
said = append(said, dropLeft(path, f)...)
|
|
if f, _ = keptRecord(path); f.Path == "" {
|
|
continue
|
|
}
|
|
}
|
|
if !f.Kept || sameUsers(f.Users, f.Want) {
|
|
continue
|
|
}
|
|
said = append(said, reconcileOne(ctx, path, f)...)
|
|
}
|
|
return said
|
|
}
|
|
|
|
// LeftFor is how long entries left on a filesystem are kept recorded, and said, while it is not back.
|
|
var LeftFor = 30 * 24 * time.Hour
|
|
|
|
// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds
|
|
// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they
|
|
// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and
|
|
// said (Leftovers).
|
|
func dropLeft(path string, f store.KeptFolder) []string {
|
|
lock := folderLock(path)
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
holding := ""
|
|
if _, err := lstatWithin(path); err == nil {
|
|
holding = mountHolding(path, MountPoints())
|
|
}
|
|
holdingID := ""
|
|
if holding != "" {
|
|
holdingID = PlacedID(holding)
|
|
}
|
|
var said []string
|
|
var still []store.KeptLeft
|
|
var mine []store.KeptUser
|
|
for _, l := range f.Left {
|
|
// On the folder's own filesystem now: the same name, or a name of an earlier build's form at the same
|
|
// mount point (which cannot be compared, and is never walked away).
|
|
same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed || l.Fs == "")
|
|
back := l.Mount == holding && (l.Placed == "" || l.Placed == holdingID)
|
|
if l.Since.IsZero() {
|
|
l.Since = time.Now().UTC()
|
|
}
|
|
if !same && !back && time.Since(l.Since) > LeftFor {
|
|
// **A filesystem not back within LeftFor is gone** (a disk replaced, a pool destroyed): what was left
|
|
// on it is forgotten, said once, rather than said unhealthy for ever.
|
|
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem not back for %s "+
|
|
"are forgotten: it is taken for gone", path, LeftFor))
|
|
continue
|
|
}
|
|
switch {
|
|
case f.Kept && same:
|
|
// Left on the filesystem the folder is kept on now: the folder's own entries.
|
|
mine = append(mine, l.Users...)
|
|
case same:
|
|
// On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's
|
|
// once it is kept there again.
|
|
still = append(still, l)
|
|
case !back:
|
|
still = append(still, l)
|
|
default:
|
|
drop := map[uint32]bool{}
|
|
for _, u := range l.Users {
|
|
drop[u.UID] = true
|
|
}
|
|
w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop))
|
|
if err != nil || w.Failed != "" {
|
|
still = append(still, l)
|
|
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
|
|
"were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed)))
|
|
continue
|
|
}
|
|
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
|
|
"were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files))
|
|
}
|
|
}
|
|
_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
|
|
k.Left = still
|
|
for _, u := range mine {
|
|
if _, has := usersOf(k.Users)[u.UID]; !has {
|
|
k.Users = append(k.Users, u)
|
|
}
|
|
}
|
|
return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0)
|
|
})
|
|
return said
|
|
}
|
|
|
|
func pathlessOr(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return pathless(err)
|
|
}
|
|
|
|
func firstOf(a, b string) string {
|
|
if a != "" {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string {
|
|
lock := folderLock(path)
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
// **Judged as an apply judges it, before every walk** (§9, §12): a disk not there leaves its mount point —
|
|
// an empty folder of root's on the filesystem below — at the path, and a walk there would write onto root's
|
|
// folder, or "drop" entries from a folder that never had them and forget the ones on the data. Nothing is
|
|
// walked then, and the record stays for the walk that can reach the data.
|
|
if v := assessFolder(ctx, path, f, false); v.err != nil || v.kind != folderKeep || v.moved {
|
|
why := "the disk that holds it is not there"
|
|
switch {
|
|
case v.err != nil:
|
|
why = pathless(v.err)
|
|
case v.kind == folderRoot:
|
|
why = RootWords(v.why)
|
|
case v.kind == folderUnknown:
|
|
why = v.why
|
|
case v.moved || v.kind == folderWaitKeep:
|
|
why = "what is at it now is not what the mesh kept there"
|
|
}
|
|
_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
|
|
k.Act = &store.KeptAct{Act: "walk", State: "waiting", Started: time.Now().UTC(),
|
|
Why: "not walked: " + why + "; walked once it is"}
|
|
return had
|
|
})
|
|
return []string{fmt.Sprintf("the kept folder %s: not walked, %s", path, why)}
|
|
}
|
|
have, want := usersOf(f.Users), usersOf(f.Want)
|
|
drop := map[uint32]bool{}
|
|
adds := false
|
|
for uid, rw := range have {
|
|
if w, ok := want[uid]; !ok || w != rw {
|
|
drop[uid] = true
|
|
}
|
|
}
|
|
for uid, rw := range want {
|
|
if h, ok := have[uid]; !ok || h != rw {
|
|
adds = true
|
|
}
|
|
}
|
|
act, visit := "add", keepVisit(want)
|
|
switch {
|
|
case len(drop) > 0 && adds:
|
|
visit = dropThenKeep(drop, want)
|
|
case len(drop) > 0:
|
|
act, visit = "drop", dropVisit(drop)
|
|
}
|
|
started := time.Now().UTC()
|
|
w, err := walkKept(path, ActLimits, machineStats, nil, visit)
|
|
rec := actRecord(act, "", "", started, w, err)
|
|
_ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool {
|
|
k.Act = rec
|
|
// Done only when every entry was reached and changed: one that failed is walked again.
|
|
if err == nil && w.Failed == "" {
|
|
k.Users = f.Want
|
|
}
|
|
return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0)
|
|
})
|
|
return []string{fmt.Sprintf("the kept folder %s: %s walk %s (%d folders and %d files changed, %d had "+
|
|
"them)%s", path, act, rec.State, rec.Folders, rec.Files, rec.Kept, orWhy(rec.Why))}
|
|
}
|
|
|
|
func orWhy(why string) string {
|
|
if why == "" {
|
|
return ""
|
|
}
|
|
return ": " + why
|
|
}
|
|
|
|
// TakeFolder is the take act (ADR 0325 §8): on the operator's warrant, the users' entries added to the folder of
|
|
// the access id and everything under it. It judges the ask against the declaration last applied and the folder as
|
|
// it is, sets the filesystem's ACL type where a ZFS dataset allows none yet, and starts the walk: what it answers
|
|
// at once, and a channel closed when the walk is through. Refused, nothing is changed.
|
|
func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warrant, by string,
|
|
run Runner) (<-chan struct{}, string, error) {
|
|
if d == nil {
|
|
return nil, "", errors.New("this engine has applied no declaration, and takes no folder it was not told of")
|
|
}
|
|
if err := ResolveRunners(ctx, d)[id]; err != nil {
|
|
return nil, "", err
|
|
}
|
|
var access *declaration.Access
|
|
for _, r := range d.Resources {
|
|
if a, ok := r.(*declaration.Access); ok && a.ID == id {
|
|
access = a
|
|
}
|
|
}
|
|
switch {
|
|
case access == nil:
|
|
return nil, "", fmt.Errorf("the access %s is not declared on this machine", id)
|
|
case filepath.Clean(access.Path) != path:
|
|
return nil, "", fmt.Errorf("the access %s is declared at another folder than the one asked", id)
|
|
}
|
|
folders := foldersOf(d.Resources, refusedPlaces(d.Resources))
|
|
want := folders[path]
|
|
users := want.list()
|
|
if len(users) == 0 {
|
|
return nil, "", errors.New("no identity uses the folder, so there is nothing to keep")
|
|
}
|
|
rec, _ := keptRecord(path)
|
|
v := assessFolder(ctx, path, rec, false)
|
|
switch {
|
|
case v.err != nil:
|
|
return nil, "", v.err
|
|
case v.kind == folderWaitDisk:
|
|
return nil, "", errors.New("the disk that holds the folder is not there")
|
|
case v.kind == folderRoot:
|
|
return nil, "", errors.New(RootWords(v.why))
|
|
case v.kind == folderMake:
|
|
return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody")
|
|
case v.unnamed:
|
|
return nil, "", errors.New(v.why + ", so a warrant could name no filesystem it was given for")
|
|
}
|
|
lock := folderLock(path)
|
|
if !lock.TryLock() {
|
|
return nil, "", errors.New("a walk of the folder is running; asked again once it is through, the act goes on")
|
|
}
|
|
said := ""
|
|
_, err := keepTop(path, usersOf(users))
|
|
if errors.Is(err, ErrNoACL) {
|
|
allowed, aerr := allowACLs(ctx, v.mount, run)
|
|
if aerr != nil {
|
|
lock.Unlock()
|
|
return nil, "", aerr
|
|
}
|
|
said = allowed + "; "
|
|
_, err = keepTop(path, usersOf(users))
|
|
}
|
|
if err != nil {
|
|
lock.Unlock()
|
|
return nil, "", err
|
|
}
|
|
started := time.Now().UTC()
|
|
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
|
|
if v.moved {
|
|
*f = movedRecord(*f, v.mount)
|
|
}
|
|
if !f.Kept {
|
|
f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil
|
|
}
|
|
f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids
|
|
// The warrant is for the filesystem it was carried out on (§8), named as a wait names it.
|
|
f.Placed, f.Fs = placeOf(v.mount)
|
|
f.TakenOn = f.Placed
|
|
f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started}
|
|
return true
|
|
})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
defer lock.Unlock()
|
|
w, err := walkKept(path, ActLimits, machineStats, nil, keepVisit(usersOf(users)))
|
|
rec := actRecord("take", warrant, by, started, w, err)
|
|
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
|
|
f.Act = rec
|
|
if err == nil {
|
|
f.Users = users
|
|
}
|
|
return true
|
|
})
|
|
}()
|
|
return done, said + "the take act started under the warrant " + warrant + ": the entries of " +
|
|
usersWords(users) + " are added to everything under the folder, following no link and entering no other " +
|
|
"filesystem", nil
|
|
}
|
|
|
|
// allowACLs lets the filesystem at mount carry ACLs, inside the take act only (ADR 0325 §8): a ZFS dataset's
|
|
// acltype set to posixacl, the dataset read from the machine's mounts. Any other filesystem is refused, naming why.
|
|
func allowACLs(ctx context.Context, mount string, run Runner) (string, error) {
|
|
source, fstype := mountSourceOf(mount)
|
|
if fstype != "zfs" || source == "" {
|
|
return "", fmt.Errorf("the filesystem that holds the folder (%s) allows no access entries, and only a ZFS "+
|
|
"dataset's can be let by the act; the operator mounts it with ACLs", orUnknown(fstype))
|
|
}
|
|
if _, err := run(ctx, "zfs", "set", "acltype=posixacl", source); err != nil {
|
|
return "", fmt.Errorf("the ZFS dataset that holds the folder could not be let carry access entries "+
|
|
"(zfs set acltype=posixacl): %w", err)
|
|
}
|
|
return "set acltype=posixacl on the ZFS dataset " + source, nil
|
|
}
|
|
|
|
func orUnknown(s string) string {
|
|
if s == "" {
|
|
return "of a type the machine does not say"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// KeptLook is what one look found of a kept folder (ADR 0325 §10).
|
|
type KeptLook struct {
|
|
// Short is every user the folder itself gives less than its right, by entry and mask together, in words.
|
|
Short []string
|
|
// Busy is a walk of the folder running: the sweep waited.
|
|
Busy bool
|
|
// Sweep is the sweep's place after this look.
|
|
Sweep store.KeptSweep
|
|
// Repaired is how many entries this look found narrowed and put back; Unrepaired how many it could not.
|
|
Repaired, Unrepaired int
|
|
// Failed is why the sweep could not look, in words with no path.
|
|
Failed string
|
|
}
|
|
|
|
// LookKept looks at a kept folder once: the folder itself judged for each user, then the sweep's bounded part of
|
|
// what is under it, from where the last look stopped, putting back what an app narrowed. The sweep's place is
|
|
// recorded; a look that ran out of its bound says so.
|
|
func LookKept(path string, users []store.KeptUser, limits WalkLimits) KeptLook {
|
|
var out KeptLook
|
|
short, err := judgeTop(path, users)
|
|
if err != nil {
|
|
out.Failed = err.Error()
|
|
return out
|
|
}
|
|
out.Short = short
|
|
lock := folderLock(path)
|
|
if !lock.TryLock() {
|
|
out.Busy = true
|
|
rec, _ := keptRecord(path)
|
|
out.Sweep = rec.Sweep
|
|
return out
|
|
}
|
|
defer lock.Unlock()
|
|
rec, _ := keptRecord(path)
|
|
sw := rec.Sweep
|
|
w, err := walkKept(path, limits, machineStats, sw.Cursor, keepVisit(usersOf(users)))
|
|
out.Repaired, out.Unrepaired = w.counts.Narrowed-w.counts.Unrepaired, w.counts.Unrepaired
|
|
sw.Read += w.counts.Read
|
|
sw.Narrowed += w.counts.Narrowed
|
|
sw.Unrepaired += w.counts.Unrepaired
|
|
switch {
|
|
case errors.Is(err, ErrWalkBound):
|
|
sw.Cursor, sw.Bound = w.Cursor, true
|
|
case err != nil:
|
|
out.Failed = err.Error()
|
|
default:
|
|
// Covered whole: the next look starts a new pass, and what this one could not put back is kept until
|
|
// the next pass is through.
|
|
sw = store.KeptSweep{Covered: time.Now().UTC(), LastUnrepaired: sw.Unrepaired}
|
|
}
|
|
out.Sweep = sw
|
|
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
|
|
f.Sweep = sw
|
|
return had
|
|
})
|
|
return out
|
|
}
|
|
|
|
// judgeTop is every user the folder itself gives less than its right, read only.
|
|
func judgeTop(path string, users []store.KeptUser) ([]string, error) {
|
|
var short []string
|
|
_, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer unix.Close(fd)
|
|
a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, u := range users {
|
|
need := needOn(u.Use == declaration.AccessReadWrite, true)
|
|
if got := effective(a, u.UID, st.Uid, uint32(st.Mode)&0o7777); got&need != need {
|
|
short = append(short, fmt.Sprintf("%s has %s of the %s it needs", u.Identity, permWords(got),
|
|
permWords(need)))
|
|
}
|
|
}
|
|
return short, nil
|
|
}
|
|
|
|
// JudgeKept reads, without changing anything, how many entries under a kept folder give a user less than its
|
|
// right, within limits: what the access says while an entry an app narrowed is not put back yet (ADR 0325 §6).
|
|
func JudgeKept(path string, users []store.KeptUser, limits WalkLimits) (WalkCounts, error) {
|
|
um := usersOf(users)
|
|
w, err := walkKept(path, limits, machineStats, nil, func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool,
|
|
error) {
|
|
a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777)
|
|
if err != nil {
|
|
return false, false, false, err
|
|
}
|
|
n, u := judgeACL(a, um, st, folder)
|
|
return false, n, u, nil
|
|
})
|
|
return w.counts, err
|
|
}
|
|
|
|
// aclProbe is what a keep wait says of its folder's filesystem and ACLs: ACLNeedsEnabling when it refuses them
|
|
// now and is a ZFS dataset the take act lets carry them, ACLUnsupported when no act can, "" when it allows
|
|
// them or the folder cannot be read (the wait says nothing then). Reads only.
|
|
func aclProbe(path, mount string) string {
|
|
_, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer unix.Close(fd)
|
|
if _, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777); !errors.Is(err, ErrNoACL) {
|
|
return ""
|
|
}
|
|
return aclWord(mountSourceOf(mount))
|
|
}
|
|
|
|
// aclWord is what a keep wait says of a filesystem that refuses ACLs now, by its mount's source and type: only a
|
|
// ZFS dataset is one the take act lets carry them.
|
|
func aclWord(source, fstype string) string {
|
|
if fstype == "zfs" && source != "" {
|
|
return ACLNeedsEnabling
|
|
}
|
|
return ACLUnsupported
|
|
}
|