mesh-host/internal/apply/library.go

package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// A library is kept usable by its group (novox/hq ADR 0328, issue 567).
//
// A library is a named tree of shared data with one home (§1). On its home the engine keeps its root and every
// sub-path its modules ask for — those folders themselves, never what is in them — in the library's group with
// group read, write and enter and the setgid bit, so what is made beneath keeps the group (§6); a missing sub-path
// is made with the root's owner and the library's group (§9). The owner, the owner's bits and others' bits are
// never changed. What the root already holds joins only on the operator's word (§7): until then the library waits
// with reason keep and nothing under it is walked. A root that is absent, or whose disk is not there, makes
// nothing, holds every container that mounts the library, and waits with reason disk (§9). Off its home a library is
// a mount the mounts module makes: nothing is kept there, and a mount that is not there holds the containers too.
//
// **When a root is the mesh's to keep without asking.** When it holds nothing yet, or is already in the library's
// group with group write and setgid (kept so before, by hand or by the mesh). Every other root waits for the take
// act, which walks it whole (TakeLibrary). Kept, it stays kept on the filesystem it was kept on — named by UUID,
// filesystem id or source (PlacedID) — and another filesystem at its root is judged afresh.

// libraryRecords is where the engine keeps what it knows of each library it is home to: the file beside the state
// in the engine, memory in a test.
type libraryRecords interface {
	All() (map[string]store.LibraryRecord, error)
	Update(name string, change func(r *store.LibraryRecord, had bool) bool) error
}

// Libraries is the engine's record of the libraries it is home to. The engine sets the file beside its state.
var Libraries libraryRecords = NewLibraryMemory()

type libraryMemory struct {
	mu sync.Mutex
	m  map[string]store.LibraryRecord
}

// NewLibraryMemory is an empty record held in memory: a test's, or a one-shot apply's.
func NewLibraryMemory() libraryRecords { return &libraryMemory{m: map[string]store.LibraryRecord{}} }

func (k *libraryMemory) All() (map[string]store.LibraryRecord, error) {
	k.mu.Lock()
	defer k.mu.Unlock()
	out := map[string]store.LibraryRecord{}
	for n, r := range k.m {
		out[n] = r
	}
	return out, nil
}

func (k *libraryMemory) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error {
	k.mu.Lock()
	defer k.mu.Unlock()
	r, had := k.m[name]
	r.Library = name
	if change(&r, had) {
		k.m[name] = r
	} else {
		delete(k.m, name)
	}
	return nil
}

// libraryRecord is the record of one library, and whether there is one. A record that cannot be read is none: the
// library is judged afresh, which waits rather than keeps.
func libraryRecord(name string) (store.LibraryRecord, bool) {
	all, err := Libraries.All()
	if err != nil {
		return store.LibraryRecord{}, false
	}
	r, had := all[name]
	return r, had
}

// libraryLocks is one lock per library, held by every change to its root and every walk under it.
var libraryLocks sync.Map

func libraryLock(name string) *sync.Mutex {
	l, _ := libraryLocks.LoadOrStore(name, &sync.Mutex{})
	return l.(*sync.Mutex)
}

// libraryWalks is every library a take walk is running under now, with the filesystem it walks: set once the walk
// has been judged and started, cleared when it ends.
var libraryWalks sync.Map

// LibraryAccess is the access a library's wait and its take act name.
func LibraryAccess(name string) string { return declaration.LibraryAccessPrefix + name }

// KeepWords and DiskWords are what a library's wait says the operator gives (ADR 0328 §8, §9): never a path.
func KeepWords(name string) string { return "let the mesh keep the library " + name }
func DiskWords(name string) string { return "mount the disk that holds the library " + name }

// The judgements of a library.
const (
	// libraryKeep is a library the mesh keeps: taken, empty when first seen, or found in shape.
	libraryKeep = "keep"
	// libraryWaitKeep is a root that holds data never taken: it waits for the operator's word.
	libraryWaitKeep = "wait-keep"
	// libraryWaitDisk is a root that is absent, or whose disk (or, off its home, whose mount) is not there.
	libraryWaitDisk = "wait-disk"
	// libraryUnknown is a root whose filesystem could not be named now: left as it is until it is.
	libraryUnknown = "unknown"
	// libraryMounted is a library off its home whose mount is there: nothing to keep.
	libraryMounted = "mounted"
)

// libraryVerdict is the judgement of one library, with what it read.
type libraryVerdict struct {
	kind string
	// how is why a kept library is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptInShape.
	how string
	// mount is the mount point that holds the root; placed and fs name its filesystem.
	mount, placed, fs string
	// moved is that another filesystem than the one the library was kept on is at its root.
	moved bool
	why   string
	err   error
}

// assessLibrary judges a library as an apply, a look and the take act all do (ADR 0328 §7, §9, §15): its group
// first, then its disk, then whether the mesh may keep it. Its words carry no path: they reach the operator's
// channel. Reads only.
func assessLibrary(ctx context.Context, l declaration.Library, rec store.LibraryRecord) libraryVerdict {
	if why := rootGroup(l.GID); why != "" {
		return libraryVerdict{err: fmt.Errorf("its group is %s, and a library's group is never a root group "+
			"(novox/hq ADR 0328 §15); nothing was made or changed", why)}
	}
	// Nowhere the machine's own trees are, as for an access (novox/hq issue 339): a library at /etc would be
	// the machine's configuration given a group.
	if err := refusePath(l.Root, placingAccess, "", ""); err != nil {
		return libraryVerdict{err: errors.New(pathless(err) + "; nothing was made or changed")}
	}
	if !l.Home {
		// Off its home the mounts module mounts it: there when its root is a mount point now.
		for _, p := range MountPoints() {
			if p == l.Root {
				return libraryVerdict{kind: libraryMounted, mount: p}
			}
		}
		return libraryVerdict{kind: libraryWaitDisk, why: "its mount is not there"}
	}
	info, err := lstatWithin(l.Root)
	switch {
	case errors.Is(err, os.ErrNotExist):
		return libraryVerdict{kind: libraryWaitDisk, why: "its root is not there"}
	case errors.Is(err, errNoAnswer):
		return libraryVerdict{kind: libraryUnknown, why: "its root does not answer: a read of it did not return " +
			"within its bound"}
	case err != nil:
		return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
	case !info.IsDir():
		return libraryVerdict{err: errors.New("its root is not a folder (a file, or a link), and a library's root " +
			"is a folder the engine reaches through no link")}
	}
	pl := placedOn(ctx, l.Root, l.Root, rec.Mount, recordedLibraryMounts(l.Library))
	if pl.Away {
		return libraryVerdict{kind: libraryWaitDisk, mount: pl.Mount, why: "the disk that holds its root is not there"}
	}
	fs := filesystemKey(pl.Mount)
	here := placedFrom(pl.Mount, fs)
	// **A filesystem that cannot be named now is unknown, never a move**: the library holds what it is — nothing
	// walked, nothing newly kept, nothing reset — and is judged again when it is named. One never kept is so never
	// kept: it fails closed.
	if here == "" {
		return libraryVerdict{kind: libraryUnknown, mount: pl.Mount,
			why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"}
	}
	moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here)
	v := libraryVerdict{mount: pl.Mount, placed: here, fs: fs, moved: moved}
	// The controller's word that the library was taken holds for the filesystem the warrant was carried out on.
	takenHere := l.Taken && (rec.KeptOn == here || (rec.KeptOn == "" && !moved))
	switch {
	case rec.Kept && rec.KeptOn == here && !moved && rec.How == store.KeptByWarrant, takenHere:
		// Taken whole on the operator's word: every sub-path is the library's.
		v.kind, v.how = libraryKeep, store.KeptByWarrant
		return v
	case rec.Kept && rec.KeptOn == here && !moved:
		// Kept without a warrant (empty, or in shape): a sub-path asked for since that holds data not in the group
		// waits for the operator's word as the root would (§7).
		v.kind, v.how = libraryKeep, rec.How
		if !pathsInShape(l, rec.Paths) {
			v.kind, v.how = libraryWaitKeep, ""
		}
		return v
	}
	facts, err := readFolder(l.Root)
	if err != nil {
		return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
	}
	switch {
	case facts.Empty:
		// Nothing there, so no sub-path either: every one is made.
		v.kind, v.how = libraryKeep, store.KeptEmpty
	case int(facts.GID) == l.GID && facts.Mode&0o2070 == 0o2070 && sampleInShape(l.Root, l.GID) &&
		pathsInShape(l, nil):
		// The root and a bounded sample beneath it already in the group with group write and setgid: data that
		// is the library's already, kept without asking. One entry of the sample out of shape asks (§7).
		v.kind, v.how = libraryKeep, store.KeptInShape
	default:
		// What is there holds data the mesh never took — a disk mounted over a root kept while it was empty below
		// it included (a move): it waits for the operator's word, and nothing under it is walked.
		v.kind = libraryWaitKeep
	}
	return v
}

// recordedLibraryMounts is every mount point recorded for another library: where a disk was once seen.
func recordedLibraryMounts(except string) []string {
	all, err := Libraries.All()
	if err != nil {
		return nil
	}
	var out []string
	for name, r := range all {
		if name != except && r.Mount != "" && r.Mount != "/" {
			out = append(out, r.Mount)
		}
	}
	return out
}

// applyLibrary is one library, applied (ADR 0328 §6, §7, §9): kept on its home when the mesh may keep it, waiting
// otherwise, and failing for a root group. Nothing is changed on the root but the group, group rwx and setgid of
// the root and its requested sub-paths, and the sub-paths made.
func applyLibrary(ctx context.Context, l declaration.Library) (Outcome, libraryVerdict, error) {
	out := Outcome{ID: LibraryAccess(l.Library), Type: "library", Target: l.Root}
	lock := libraryLock(l.Library)
	if !lock.TryLock() {
		// **A take walk is running (up to its bound, half an hour): the apply does not wait for it.** The library is
		// the walk's until it is through; it keeps it, and its containers are not held for it. Only a walk that
		// has started, on the filesystem it judged: anything else holding the lock (a take act about to be
		// refused, say) is short, and the apply waits for it and judges the library as always.
		if on, walking := libraryWalks.Load(l.Library); walking {
			out.Action = "unchanged"
			out.Detail = "the take act is setting its group under its root now; this apply leaves the library to it"
			return out, libraryVerdict{kind: libraryKeep, how: store.KeptByWarrant, placed: on.(string)}, nil
		}
		lock.Lock()
	}
	defer lock.Unlock()
	rec, _ := libraryRecord(l.Library)
	v := assessLibrary(ctx, l, rec)
	if v.err != nil {
		return out, v, v.err
	}
	switch v.kind {
	case libraryMounted:
		out.Action, out.Detail = "unchanged", "not its home: mounted, and nothing is kept here"
		return out, v, nil
	case libraryWaitDisk:
		out.Action = "waiting"
		out.Detail = v.why + ": nothing is made, and no container that mounts it is created until it is there " +
			"(novox/hq ADR 0328 §9); waits for the operator to " + DiskWords(l.Library)
		return out, v, nil
	case libraryUnknown:
		out.Action = "waiting"
		out.Detail = v.why + ": the library is left exactly as it is, and judged again on the next apply"
		return out, v, nil
	case libraryWaitKeep:
		_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, had bool) bool {
			if !had || v.moved {
				r.Root, r.Mount, r.Placed, r.Fs = l.Root, v.mount, v.placed, v.fs
			}
			if v.moved {
				// Kept on another filesystem: what was kept there no longer holds here.
				r.Kept, r.How, r.Cursor = false, "", nil
			}
			return true
		})
		out.Action = "waiting"
		out.Detail = "its root holds data the mesh has not taken: nothing is changed under it until the operator's " +
			"warrant (novox/hq ADR 0328 §7); waits for the operator to " + KeepWords(l.Library)
		return out, v, nil
	}
	changed, made, err := keepFolders(l.Root, l.Paths, l.GID)
	if err != nil {
		return out, v, err
	}
	_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, _ bool) bool {
		if !r.Kept || r.KeptOn != v.placed {
			r.How, r.At = v.how, time.Now().UTC()
		}
		r.Root, r.Mount, r.Placed, r.Fs, r.GID = l.Root, v.mount, v.placed, v.fs, l.GID
		r.Kept, r.KeptOn = true, v.placed
		r.Paths = unionPaths(r.Paths, l.Paths)
		return true
	})
	out.Action = "unchanged"
	if changed+made > 0 {
		out.Action = "updated"
	}
	out.Detail = fmt.Sprintf("kept in %s: its root and %d sub-path(s) with group rwx and setgid, %d made; owners "+
		"and everyone else's rights unchanged (novox/hq ADR 0328 §6)", l.Group, len(l.Paths), made)
	return out, v, nil
}

// heldRoots is every library root whose containers are held: its disk, or its mount, is not there (§9).
func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) map[string]string {
	out := map[string]string{}
	for _, l := range libs {
		if v, ok := verdicts[l.Library]; ok && v.kind == libraryWaitDisk {
			out[l.Root] = l.Library
		}
	}
	return out
}

// mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held.
func mountsHeld(c *declaration.Container, held map[string]string) string {
	for _, v := range c.Volumes {
		src := mountSource(v)
		if !strings.HasPrefix(src, "/") {
			continue
		}
		for root, name := range held {
			if atOrBelow(filepath.Clean(src), root) {
				return name
			}
		}
	}
	return ""
}

// WaitingOnLibrary is every container of the declaration that waits for a library whose disk or mount is not
// there, read now: not created by the apply, so not judged alive or dead — the library says why.
func WaitingOnLibrary(d *declaration.Declaration) map[string]bool {
	out := map[string]bool{}
	if d == nil || len(d.Libraries) == 0 {
		return out
	}
	verdicts := map[string]libraryVerdict{}
	for _, l := range d.Libraries {
		rec, _ := libraryRecord(l.Library)
		verdicts[l.Library] = assessLibrary(context.Background(), l, rec)
	}
	held := heldRoots(verdicts, d.Libraries)
	for _, r := range d.Resources {
		if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" {
			out[c.ID] = true
		}
	}
	return out
}

// The judgements of a library as a look says them.
const (
	VerdictKeep     = "keep"
	VerdictMounted  = "mounted"
	VerdictWaitKeep = "wait-keep"
	VerdictWaitDisk = "wait-disk"
	VerdictUnknown  = "unknown"
	VerdictBroken   = "broken"
)

// LibraryJudgement is a look's judgement of a library, and why, in words with no path; Placed names its root's
// filesystem for a keep wait, with no path (PlacedID).
type LibraryJudgement struct {
	Kind, Why, Placed string
	// Act is the last walk under its root.
	Act *store.KeptAct
}

// JudgeLibrary judges a library for a look, read only.
func JudgeLibrary(ctx context.Context, l declaration.Library) LibraryJudgement {
	rec, _ := libraryRecord(l.Library)
	v := assessLibrary(ctx, l, rec)
	j := LibraryJudgement{Act: rec.Act}
	switch {
	case v.err != nil:
		j.Kind, j.Why = VerdictBroken, pathless(v.err)
	case v.kind == libraryWaitKeep:
		j.Kind, j.Placed = VerdictWaitKeep, v.placed
	case v.kind == libraryWaitDisk:
		j.Kind, j.Why = VerdictWaitDisk, v.why
	case v.kind == libraryUnknown:
		j.Kind, j.Why = VerdictUnknown, v.why
	case v.kind == libraryMounted:
		j.Kind = VerdictMounted
	default:
		j.Kind = VerdictKeep
	}
	return j
}

// joinRunner puts a library's tool runner's account in its group (ADR 0328 §11).
func joinRunner(ctx context.Context, sys system.System, l declaration.Library, known *store.State,
	run Runner) (bool, string, error) {
	if l.Runner == "" {
		return false, "", nil
	}
	return joinGroup(ctx, sys, l.Runner, l.Group, store.Membership{Library: l.Library}, known, run)
}

// IdentityChange is what a tool runner joining a module's own group changes (ADR 0334 §2): `identity:<module>`,
// which the tool runner's unit names under restart-on, as it names `library:<name>`.
func IdentityChange(module string) string { return "identity:" + module }

// joinGroup puts a tool runner's account in a group (ADR 0328 §11, ADR 0334 §2), as ADR 0252 puts an account in a
// group: recorded only when the mesh put it there, so one it was in before is never taken back. The account must be
// here and never root's: uid 0, or a root group as its primary (§15). Whether it moved, and what was said.
func joinGroup(ctx context.Context, sys system.System, account, group string, of store.Membership, known *store.State,
	run Runner) (bool, string, error) {
	who, found, err := lookUp(ctx, run, "passwd", account)
	switch {
	case err != nil:
		return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
	case !found:
		return false, "", fmt.Errorf("the tool runner's account %s is not on this machine, so it joins no group", account)
	}
	if err := rootIdentity(who.id, who.prim); err != nil {
		return false, "", fmt.Errorf("the tool runner's account %s joins no group: %w", account, err)
	}
	in, err := system.GroupsOf(ctx, system.Runner(run), account)
	if err != nil {
		return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
	}
	if setOf(in)[group] {
		return false, "", nil
	}
	if err := sys.AddUserToGroup(ctx, system.Runner(run), account, group); err != nil {
		return false, "", err
	}
	for _, m := range known.Members {
		if m.Account == account && m.Group == group {
			return true, fmt.Sprintf("put the tool runner's %s back in %s", account, group), nil
		}
	}
	of.Account, of.Group = account, group
	known.Members = append(known.Members, of)
	return true, fmt.Sprintf("put the tool runner's %s in %s; it holds it once restarted", account, group), nil
}

// leaveRunners takes every tool runner's account the mesh put in a library's group, or a module's own group, out of
// it once no declared library or identity puts it there and no declared user resource asks for it (ADR 0252 §3). Never fatal: what could not be
// left stays recorded, said, for the next apply.
func leaveRunners(ctx context.Context, sys system.System, libs []declaration.Library, ids []declaration.Identity,
	known *store.State, run Runner, wanted Wanted) []Outcome {
	declared := map[string]bool{}
	for _, l := range libs {
		if l.Runner != "" {
			declared[l.Runner+" "+l.Group] = true
		}
	}
	for _, id := range ids {
		if id.Runner != "" {
			declared[id.Runner+" "+id.Name] = true
		}
	}
	var out []Outcome
	var keep []store.Membership
	for _, m := range known.Members {
		if declared[m.Account+" "+m.Group] {
			keep = append(keep, m)
			continue
		}
		o := Outcome{ID: LibraryAccess(m.Library) + " runner", Type: "library", Target: m.Account}
		if m.Module != "" {
			o.ID, o.Type = "identity "+declaration.IdentityPrefix+m.Module+" runner", "identity"
		}
		if others := wanted.othersAsk(m.Account, m.Group, ""); len(others) > 0 {
			o.Action, o.Detail = "forgotten", fmt.Sprintf("left %s in %s, which %s asks for", m.Account, m.Group,
				strings.Join(others, ", "))
			out = append(out, o)
			continue
		}
		gone, why := leaveGroup(ctx, sys, m.Account, m.Group, run)
		o.Action, o.Detail = "restored", m.Account+" "+why
		if !gone {
			o.Action = "waiting"
			keep = append(keep, m)
		}
		out = append(out, o)
	}
	known.Members = keep
	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
	return out
}

// pathsInShape is whether every requested sub-path not kept before is absent (it is made), empty, or already in the
// group with group write and setgid, it and a bounded sample beneath it: what may be kept without the operator's
// word (§7). One that cannot be read is not.
func pathsInShape(l declaration.Library, kept []string) bool {
	done := map[string]bool{}
	for _, p := range kept {
		done[p] = true
	}
	for _, p := range l.Paths {
		if done[p] {
			continue
		}
		path := filepath.Join(l.Root, p)
		if _, err := lstatWithin(path); errors.Is(err, os.ErrNotExist) {
			continue
		}
		facts, err := readFolder(path)
		if err != nil {
			return false
		}
		if facts.Empty {
			continue
		}
		if int(facts.GID) != l.GID || facts.Mode&0o2070 != 0o2070 || !sampleInShape(path, l.GID) {
			return false
		}
	}
	return true
}

// unionPaths is a with every path of b not in it.
func unionPaths(a, b []string) []string {
	have := map[string]bool{}
	for _, p := range a {
		have[p] = true
	}
	for _, p := range b {
		if !have[p] {
			a = append(a, p)
			have[p] = true
		}
	}
	sort.Strings(a)
	return a
}