package apply import ( "context" "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // A library is kept usable by its group (novox/hq ADR 0328, issue 567). // // A library is a named tree of shared data with one home (§1). On its home the engine keeps its root and every // sub-path its modules ask for — those folders themselves, never what is in them — in the library's group with // group read, write and enter and the setgid bit, so what is made beneath keeps the group (§6); a missing sub-path // is made with the root's owner and the library's group (§9). The owner, the owner's bits and others' bits are // never changed. What the root already holds joins only on the operator's word (§7): until then the library waits // with reason keep and nothing under it is walked. A root that is absent, or whose disk is not there, makes // nothing, holds every container that mounts the library, and waits with reason disk (§9). Off its home a library is // a mount the mounts module makes: nothing is kept there, and a mount that is not there holds the containers too. // // **When a root is the mesh's to keep without asking.** When it holds nothing yet, or is already in the library's // group with group write and setgid (kept so before, by hand or by the mesh). Every other root waits for the take // act, which walks it whole (TakeLibrary). Kept, it stays kept on the filesystem it was kept on — named by UUID, // filesystem id or source (PlacedID) — and another filesystem at its root is judged afresh. // libraryRecords is where the engine keeps what it knows of each library it is home to: the file beside the state // in the engine, memory in a test. type libraryRecords interface { All() (map[string]store.LibraryRecord, error) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error } // Libraries is the engine's record of the libraries it is home to. The engine sets the file beside its state. var Libraries libraryRecords = NewLibraryMemory() type libraryMemory struct { mu sync.Mutex m map[string]store.LibraryRecord } // NewLibraryMemory is an empty record held in memory: a test's, or a one-shot apply's. func NewLibraryMemory() libraryRecords { return &libraryMemory{m: map[string]store.LibraryRecord{}} } func (k *libraryMemory) All() (map[string]store.LibraryRecord, error) { k.mu.Lock() defer k.mu.Unlock() out := map[string]store.LibraryRecord{} for n, r := range k.m { out[n] = r } return out, nil } func (k *libraryMemory) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error { k.mu.Lock() defer k.mu.Unlock() r, had := k.m[name] r.Library = name if change(&r, had) { k.m[name] = r } else { delete(k.m, name) } return nil } // libraryRecord is the record of one library, and whether there is one. A record that cannot be read is none: the // library is judged afresh, which waits rather than keeps. func libraryRecord(name string) (store.LibraryRecord, bool) { all, err := Libraries.All() if err != nil { return store.LibraryRecord{}, false } r, had := all[name] return r, had } // libraryLocks is one lock per library, held by every change to its root and every walk under it. var libraryLocks sync.Map func libraryLock(name string) *sync.Mutex { l, _ := libraryLocks.LoadOrStore(name, &sync.Mutex{}) return l.(*sync.Mutex) } // libraryWalks is every library a take walk is running under now, with the filesystem it walks: set once the walk // has been judged and started, cleared when it ends. var libraryWalks sync.Map // LibraryAccess is the access a library's wait and its take act name. func LibraryAccess(name string) string { return declaration.LibraryAccessPrefix + name } // KeepWords and DiskWords are what a library's wait says the operator gives (ADR 0328 §8, §9): never a path. func KeepWords(name string) string { return "let the mesh keep the library " + name } func DiskWords(name string) string { return "mount the disk that holds the library " + name } // The judgements of a library. const ( // libraryKeep is a library the mesh keeps: taken, empty when first seen, or found in shape. libraryKeep = "keep" // libraryWaitKeep is a root that holds data never taken: it waits for the operator's word. libraryWaitKeep = "wait-keep" // libraryWaitDisk is a root that is absent, or whose disk (or, off its home, whose mount) is not there. libraryWaitDisk = "wait-disk" // libraryUnknown is a root whose filesystem could not be named now: left as it is until it is. libraryUnknown = "unknown" // libraryMounted is a library off its home whose mount is there: nothing to keep. libraryMounted = "mounted" ) // libraryVerdict is the judgement of one library, with what it read. type libraryVerdict struct { kind string // how is why a kept library is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptInShape. how string // mount is the mount point that holds the root; placed and fs name its filesystem. mount, placed, fs string // moved is that another filesystem than the one the library was kept on is at its root. moved bool why string err error } // assessLibrary judges a library as an apply, a look and the take act all do (ADR 0328 §7, §9, §15): its group // first, then its disk, then whether the mesh may keep it. Its words carry no path: they reach the operator's // channel. Reads only. func assessLibrary(ctx context.Context, l declaration.Library, rec store.LibraryRecord) libraryVerdict { if why := rootGroup(l.GID); why != "" { return libraryVerdict{err: fmt.Errorf("its group is %s, and a library's group is never a root group "+ "(novox/hq ADR 0328 §15); nothing was made or changed", why)} } // Nowhere the machine's own trees are, as for an access (novox/hq issue 339): a library at /etc would be // the machine's configuration given a group. if err := refusePath(l.Root, placingAccess, "", ""); err != nil { return libraryVerdict{err: errors.New(pathless(err) + "; nothing was made or changed")} } if !l.Home { // Off its home the mounts module mounts it: there when its root is a mount point now. for _, p := range MountPoints() { if p == l.Root { return libraryVerdict{kind: libraryMounted, mount: p} } } return libraryVerdict{kind: libraryWaitDisk, why: "its mount is not there"} } info, err := lstatWithin(l.Root) switch { case errors.Is(err, os.ErrNotExist): return libraryVerdict{kind: libraryWaitDisk, why: "its root is not there"} case errors.Is(err, errNoAnswer): return libraryVerdict{kind: libraryUnknown, why: "its root does not answer: a read of it did not return " + "within its bound"} case err != nil: return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))} case !info.IsDir(): return libraryVerdict{err: errors.New("its root is not a folder (a file, or a link), and a library's root " + "is a folder the engine reaches through no link")} } pl := placedOn(ctx, l.Root, l.Root, rec.Mount, recordedLibraryMounts(l.Library)) if pl.Away { return libraryVerdict{kind: libraryWaitDisk, mount: pl.Mount, why: "the disk that holds its root is not there"} } fs := filesystemKey(pl.Mount) here := placedFrom(pl.Mount, fs) // **A filesystem that cannot be named now is unknown, never a move**: the library holds what it is — nothing // walked, nothing newly kept, nothing reset — and is judged again when it is named. One never kept is so never // kept: it fails closed. if here == "" { return libraryVerdict{kind: libraryUnknown, mount: pl.Mount, why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"} } moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here) v := libraryVerdict{mount: pl.Mount, placed: here, fs: fs, moved: moved} // The controller's word that the library was taken holds for the filesystem the warrant was carried out on. takenHere := l.Taken && (rec.KeptOn == here || (rec.KeptOn == "" && !moved)) switch { case rec.Kept && rec.KeptOn == here && !moved && rec.How == store.KeptByWarrant, takenHere: // Taken whole on the operator's word: every sub-path is the library's. v.kind, v.how = libraryKeep, store.KeptByWarrant return v case rec.Kept && rec.KeptOn == here && !moved: // Kept without a warrant (empty, or in shape): a sub-path asked for since that holds data not in the group // waits for the operator's word as the root would (§7). v.kind, v.how = libraryKeep, rec.How if !pathsInShape(l, rec.Paths) { v.kind, v.how = libraryWaitKeep, "" } return v } facts, err := readFolder(l.Root) if err != nil { return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))} } switch { case facts.Empty: // Nothing there, so no sub-path either: every one is made. v.kind, v.how = libraryKeep, store.KeptEmpty case int(facts.GID) == l.GID && facts.Mode&0o2070 == 0o2070 && sampleInShape(l.Root, l.GID) && pathsInShape(l, nil): // The root and a bounded sample beneath it already in the group with group write and setgid: data that // is the library's already, kept without asking. One entry of the sample out of shape asks (§7). v.kind, v.how = libraryKeep, store.KeptInShape default: // What is there holds data the mesh never took — a disk mounted over a root kept while it was empty below // it included (a move): it waits for the operator's word, and nothing under it is walked. v.kind = libraryWaitKeep } return v } // recordedLibraryMounts is every mount point recorded for another library: where a disk was once seen. func recordedLibraryMounts(except string) []string { all, err := Libraries.All() if err != nil { return nil } var out []string for name, r := range all { if name != except && r.Mount != "" && r.Mount != "/" { out = append(out, r.Mount) } } return out } // applyLibrary is one library, applied (ADR 0328 §6, §7, §9): kept on its home when the mesh may keep it, waiting // otherwise, and failing for a root group. Nothing is changed on the root but the group, group rwx and setgid of // the root and its requested sub-paths, and the sub-paths made. func applyLibrary(ctx context.Context, l declaration.Library) (Outcome, libraryVerdict, error) { out := Outcome{ID: LibraryAccess(l.Library), Type: "library", Target: l.Root} lock := libraryLock(l.Library) if !lock.TryLock() { // **A take walk is running (up to its bound, half an hour): the apply does not wait for it.** The library is // the walk's until it is through; it keeps it, and its containers are not held for it. Only a walk that // has started, on the filesystem it judged: anything else holding the lock (a take act about to be // refused, say) is short, and the apply waits for it and judges the library as always. if on, walking := libraryWalks.Load(l.Library); walking { out.Action = "unchanged" out.Detail = "the take act is setting its group under its root now; this apply leaves the library to it" return out, libraryVerdict{kind: libraryKeep, how: store.KeptByWarrant, placed: on.(string)}, nil } lock.Lock() } defer lock.Unlock() rec, _ := libraryRecord(l.Library) v := assessLibrary(ctx, l, rec) if v.err != nil { return out, v, v.err } switch v.kind { case libraryMounted: out.Action, out.Detail = "unchanged", "not its home: mounted, and nothing is kept here" return out, v, nil case libraryWaitDisk: out.Action = "waiting" out.Detail = v.why + ": nothing is made, and no container that mounts it is created until it is there " + "(novox/hq ADR 0328 §9); waits for the operator to " + DiskWords(l.Library) return out, v, nil case libraryUnknown: out.Action = "waiting" out.Detail = v.why + ": the library is left exactly as it is, and judged again on the next apply" return out, v, nil case libraryWaitKeep: _ = Libraries.Update(l.Library, func(r *store.LibraryRecord, had bool) bool { if !had || v.moved { r.Root, r.Mount, r.Placed, r.Fs = l.Root, v.mount, v.placed, v.fs } if v.moved { // Kept on another filesystem: what was kept there no longer holds here. r.Kept, r.How, r.Cursor = false, "", nil } return true }) out.Action = "waiting" out.Detail = "its root holds data the mesh has not taken: nothing is changed under it until the operator's " + "warrant (novox/hq ADR 0328 §7); waits for the operator to " + KeepWords(l.Library) return out, v, nil } changed, made, err := keepFolders(l.Root, l.Paths, l.GID) if err != nil { return out, v, err } _ = Libraries.Update(l.Library, func(r *store.LibraryRecord, _ bool) bool { if !r.Kept || r.KeptOn != v.placed { r.How, r.At = v.how, time.Now().UTC() } r.Root, r.Mount, r.Placed, r.Fs, r.GID = l.Root, v.mount, v.placed, v.fs, l.GID r.Kept, r.KeptOn = true, v.placed r.Paths = unionPaths(r.Paths, l.Paths) return true }) out.Action = "unchanged" if changed+made > 0 { out.Action = "updated" } out.Detail = fmt.Sprintf("kept in %s: its root and %d sub-path(s) with group rwx and setgid, %d made; owners "+ "and everyone else's rights unchanged (novox/hq ADR 0328 §6)", l.Group, len(l.Paths), made) return out, v, nil } // heldRoots is every library root whose containers are held: its disk, or its mount, is not there (§9). func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) map[string]string { out := map[string]string{} for _, l := range libs { if v, ok := verdicts[l.Library]; ok && v.kind == libraryWaitDisk { out[l.Root] = l.Library } } return out } // mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held. func mountsHeld(c *declaration.Container, held map[string]string) string { for _, v := range c.Volumes { src := mountSource(v) if !strings.HasPrefix(src, "/") { continue } for root, name := range held { if atOrBelow(filepath.Clean(src), root) { return name } } } return "" } // WaitingOnLibrary is every container of the declaration that waits for a library whose disk or mount is not // there, read now: not created by the apply, so not judged alive or dead — the library says why. func WaitingOnLibrary(d *declaration.Declaration) map[string]bool { out := map[string]bool{} if d == nil || len(d.Libraries) == 0 { return out } verdicts := map[string]libraryVerdict{} for _, l := range d.Libraries { rec, _ := libraryRecord(l.Library) verdicts[l.Library] = assessLibrary(context.Background(), l, rec) } held := heldRoots(verdicts, d.Libraries) for _, r := range d.Resources { if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" { out[c.ID] = true } } return out } // The judgements of a library as a look says them. const ( VerdictKeep = "keep" VerdictMounted = "mounted" VerdictWaitKeep = "wait-keep" VerdictWaitDisk = "wait-disk" VerdictUnknown = "unknown" VerdictBroken = "broken" ) // LibraryJudgement is a look's judgement of a library, and why, in words with no path; Placed names its root's // filesystem for a keep wait, with no path (PlacedID). type LibraryJudgement struct { Kind, Why, Placed string // Act is the last walk under its root. Act *store.KeptAct } // JudgeLibrary judges a library for a look, read only. func JudgeLibrary(ctx context.Context, l declaration.Library) LibraryJudgement { rec, _ := libraryRecord(l.Library) v := assessLibrary(ctx, l, rec) j := LibraryJudgement{Act: rec.Act} switch { case v.err != nil: j.Kind, j.Why = VerdictBroken, pathless(v.err) case v.kind == libraryWaitKeep: j.Kind, j.Placed = VerdictWaitKeep, v.placed case v.kind == libraryWaitDisk: j.Kind, j.Why = VerdictWaitDisk, v.why case v.kind == libraryUnknown: j.Kind, j.Why = VerdictUnknown, v.why case v.kind == libraryMounted: j.Kind = VerdictMounted default: j.Kind = VerdictKeep } return j } // joinRunner puts a library's tool runner's account in its group (ADR 0328 §11), as ADR 0252 puts an account in a // group: recorded only when the mesh put it there, so one it was in before is never taken back. Whether it moved. func joinRunner(ctx context.Context, sys system.System, l declaration.Library, known *store.State, run Runner) (bool, string, error) { if l.Runner == "" { return false, "", nil } in, err := system.GroupsOf(ctx, system.Runner(run), l.Runner) if err != nil { return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", l.Runner, err) } if setOf(in)[l.Group] { return false, "", nil } if err := sys.AddUserToGroup(ctx, system.Runner(run), l.Runner, l.Group); err != nil { return false, "", err } for _, m := range known.Members { if m.Account == l.Runner && m.Group == l.Group { return true, fmt.Sprintf("put the tool runner's %s back in %s", l.Runner, l.Group), nil } } known.Members = append(known.Members, store.Membership{Account: l.Runner, Group: l.Group, Library: l.Library}) return true, fmt.Sprintf("put the tool runner's %s in %s; it holds it once restarted", l.Runner, l.Group), nil } // leaveRunners takes every tool runner's account the mesh put in a library's group out of it once no declared // library puts it there and no declared user resource asks for it (ADR 0252 §3). Never fatal: what could not be // left stays recorded, said, for the next apply. func leaveRunners(ctx context.Context, sys system.System, libs []declaration.Library, known *store.State, run Runner, wanted Wanted) []Outcome { declared := map[string]bool{} for _, l := range libs { if l.Runner != "" { declared[l.Runner+" "+l.Group] = true } } var out []Outcome var keep []store.Membership for _, m := range known.Members { if declared[m.Account+" "+m.Group] { keep = append(keep, m) continue } o := Outcome{ID: LibraryAccess(m.Library) + " runner", Type: "library", Target: m.Account} if others := wanted.othersAsk(m.Account, m.Group, ""); len(others) > 0 { o.Action, o.Detail = "forgotten", fmt.Sprintf("left %s in %s, which %s asks for", m.Account, m.Group, strings.Join(others, ", ")) out = append(out, o) continue } gone, why := leaveGroup(ctx, sys, m.Account, m.Group, run) o.Action, o.Detail = "restored", m.Account+" "+why if !gone { o.Action = "waiting" keep = append(keep, m) } out = append(out, o) } known.Members = keep sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out } // pathsInShape is whether every requested sub-path not kept before is absent (it is made), empty, or already in the // group with group write and setgid, it and a bounded sample beneath it: what may be kept without the operator's // word (§7). One that cannot be read is not. func pathsInShape(l declaration.Library, kept []string) bool { done := map[string]bool{} for _, p := range kept { done[p] = true } for _, p := range l.Paths { if done[p] { continue } path := filepath.Join(l.Root, p) if _, err := lstatWithin(path); errors.Is(err, os.ErrNotExist) { continue } facts, err := readFolder(path) if err != nil { return false } if facts.Empty { continue } if int(facts.GID) != l.GID || facts.Mode&0o2070 != 0o2070 || !sampleInShape(path, l.GID) { return false } } return true } // unionPaths is a with every path of b not in it. func unionPaths(a, b []string) []string { have := map[string]bool{} for _, p := range a { have[p] = true } for _, p := range b { if !have[p] { a = append(a, p) have[p] = true } } sort.Strings(a) return a }