diff --git a/cmd/mesh-host/grants.go b/cmd/mesh-host/grants.go deleted file mode 100644 index 892ca57..0000000 --- a/cmd/mesh-host/grants.go +++ /dev/null @@ -1,181 +0,0 @@ -package main - -import ( - "sort" - "strings" - "sync" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/link" -) - -// A read-write access whose group cannot write is never quiet (novox/hq ADR 0312 §2). -// -// The engine grants a read-write access's path group to the identities the module's code runs as, and changes -// nothing on the path: whether its group may write is the operator's, as the data is. So an access whose path's -// group has no write permission is stated, on every look, as its module's resource of kind `access`, unhealthy, -// saying so and that the operator decides — and one owned by a group that is never granted (§4) is stated -// unhealthy saying that. The controller raises it as that module's condition on two statements in a row and -// clears it on the first healthy one, as it does every resource's health (ADR 0240). The path is the target, -// never in the reason: the controller says a reason on the operator's channel, which withholds a path (ADR 0234 -// §6). - -// grantedAccess is one read-write access that grants, as the judge reads it. -type grantedAccess struct { - module, id, path string -} - -// accessJudge reads, on every look, whether each read-write access that grants may write its path through its -// group. Safe for the apply and the looking loop at once. -type accessJudge struct { - mu sync.Mutex - accesses []grantedAccess - // was is each access's last verdict, with since when and how many looks in a row found it unhealthy. - was map[string]link.ResourceHealth - - // reading guards inFlight: every path whose read has not returned yet. - reading sync.Mutex - inFlight map[string]bool -} - -// accesses is the one access judge on this machine. -var accesses = newAccessJudge() - -func newAccessJudge() *accessJudge { - return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}} -} - -// set is what the declaration just applied asks: every read-write access of a module that names a grant, not -// held as found. An access no longer declared is forgotten. -func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) { - var as []grantedAccess - if d != nil { - for _, r := range d.Resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite || held[a.ID] { - continue - } - at := strings.LastIndex(a.ID, ".") - if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { - continue - } - as = append(as, grantedAccess{module: a.ID[:at], id: a.ID, path: a.Path}) - } - } - sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id }) - j.mu.Lock() - defer j.mu.Unlock() - j.accesses = as - declared := map[string]bool{} - for _, a := range as { - declared[a.id] = true - } - for id := range j.was { - if !declared[id] { - delete(j.was, id) - } - } -} - -// lookBound is how long a look waits for one accessed path to answer. A hard network mount whose server is down -// blocks a stat for ever, and one access must never stop every health statement this machine makes. -var lookBound = 5 * time.Second - -// pathGroupBounded reads a path's group within lookBound, outside any lock. A read that has not answered is said -// so; while it still hangs, later looks do not start another on the same path, so a dead mount costs one blocked -// read, not one per look. -func (j *accessJudge) pathGroupBounded(path string) (apply.PathGroup, bool, error) { - j.reading.Lock() - if j.inFlight[path] { - j.reading.Unlock() - return apply.PathGroup{}, false, nil - } - j.inFlight[path] = true - j.reading.Unlock() - type read struct { - g apply.PathGroup - err error - } - done := make(chan read, 1) - go func() { - g, err := apply.GroupOfPath(path) - j.reading.Lock() - delete(j.inFlight, path) - j.reading.Unlock() - done <- read{g, err} - }() - select { - case r := <-done: - return r.g, true, r.err - case <-time.After(lookBound): - return apply.PathGroup{}, false, nil - } -} - -// look reads every access now: healthy when its group may write and may be granted, unhealthy saying why -// otherwise. Whether any verdict changed since the last look, for the loop to say it. The paths are read with no -// lock held, each within lookBound. -func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { - j.mu.Lock() - as := append([]grantedAccess(nil), j.accesses...) - j.mu.Unlock() - - type verdict struct{ state, reason string } - read := make([]verdict, len(as)) - for i, a := range as { - state, reason := link.StateHealthy, "" - g, answered, err := j.pathGroupBounded(a.path) - switch { - case !answered: - state, reason = link.StateUnhealthy, "its path does not answer: reading which group owns it did not "+ - "return within "+lookBound.String() - case err != nil: - state, reason = link.StateUnhealthy, "its group cannot be read: "+strings.ReplaceAll(err.Error(), a.path, "the path") - case apply.GrantRefusal(g) != nil: - state, reason = link.StateUnhealthy, "its group is never granted: "+apply.GrantRefusal(g).Error() - case !g.Writable: - state, reason = link.StateUnhealthy, apply.CannotWriteWords(g) - } - read[i] = verdict{state, reason} - } - - j.mu.Lock() - defer j.mu.Unlock() - changed := false - out := make([]link.ResourceHealth, 0, len(as)) - for i, a := range as { - state, reason := read[i].state, read[i].reason - v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path, - State: state, Reason: reason, Since: now.UTC()} - if was, had := j.was[a.id]; had && was.State == state && was.Reason == reason { - v.Since = was.Since - v.Streak = was.Streak - } else { - changed = true - } - if state != link.StateHealthy { - v.Streak++ - } - j.was[a.id] = v - out = append(out, v) - } - return out, changed -} - -// withAccesses adds to a statement every read-write access that grants, as its module's resource of kind access. -func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health { - h.Resources = append(h.Resources, vs...) - return h -} - -// accessesHealthy says no access in vs is anything but healthy. -func accessesHealthy(vs []link.ResourceHealth) bool { - for _, v := range vs { - if v.State != link.StateHealthy { - return false - } - } - return true -} diff --git a/cmd/mesh-host/grants_test.go b/cmd/mesh-host/grants_test.go deleted file mode 100644 index 27f1227..0000000 --- a/cmd/mesh-host/grants_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package main - -import ( - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/link" -) - -// A read-write access whose path's group cannot write is stated unhealthy on every look, saying so and that the -// operator decides, and one whose group is never granted says that (novox/hq ADR 0312 §2, §4). The path is the -// target and never in the reason (ADR 0234 §6); a read access, granting nothing, is not stated at all. -func TestAnAccessWhoseGroupCannotWriteIsStatedUnhealthyWithTheOperatorsRemedy(t *testing.T) { - spool, library, shared, read := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() - groups := map[string]apply.PathGroup{ - spool: {GID: 1002, Name: "downloads", Writable: false}, - library: {GID: 1001, Name: "media", Writable: true}, - shared: {GID: 0, Name: "root", Writable: true}, - } - was := apply.GroupOfPath - apply.GroupOfPath = func(path string) (apply.PathGroup, error) { return groups[path], nil } - t.Cleanup(func() { apply.GroupOfPath = was }) - wasRoot := apply.RootGroupIDs - apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} } - t.Cleanup(func() { apply.RootGroupIDs = wasRoot }) - - d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ - {"id":"nzbget.downloads","type":"access","path":"` + spool + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"importer.shared","type":"access","path":"` + shared + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"player.music","type":"access","path":"` + read + `","mode":"read"}]}`)) - if err != nil { - t.Fatal(err) - } - j := newAccessJudge() - j.set(d, nil) - at := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) - vs, changed := j.look(at) - if !changed || len(vs) != 3 { - t.Fatalf("stated %d accesses (changed %v), want the three read-write ones: %+v", len(vs), changed, vs) - } - got := map[string]link.ResourceHealth{} - for _, v := range vs { - got[v.Resource] = v - if v.Kind != link.KindAccess || strings.Contains(v.Reason, v.Target) { - t.Errorf("%s is stated as %q with its path in the reason %q", v.Resource, v.Kind, v.Reason) - } - } - if v := got["nzbget.downloads"]; v.State != link.StateUnhealthy || v.Module != "nzbget" || v.Target != spool || - !strings.Contains(v.Reason, "cannot write") || !strings.Contains(v.Reason, "operator's to decide") || v.Streak != 1 { - t.Errorf("an access whose group cannot write is stated %+v", v) - } - if v := got["importer.shared"]; v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "never granted") { - t.Errorf("an access of the root group is stated %+v", v) - } - if v := got["importer.music"]; v.State != link.StateHealthy { - t.Errorf("an access whose group may write is stated %+v", v) - } - if h := withAccesses(&link.Health{}, vs); accessesHealthy(vs) || len(h.Resources) != 3 { - t.Errorf("the statement does not carry the accesses as unhealthy: %+v", h.Resources) - } - - // Looked at again: the same verdict, its since kept and its streak counted. - vs, changed = j.look(at.Add(time.Minute)) - for _, v := range vs { - if v.Resource == "nzbget.downloads" && (changed || v.Streak != 2 || !v.Since.Equal(at)) { - t.Errorf("the second look is %+v (changed %v)", v, changed) - } - } - // The operator gives the group write: healthy at the next look, and nothing the engine did made it so. - groups[spool] = apply.PathGroup{GID: 1002, Name: "downloads", Writable: true} - vs, _ = j.look(at.Add(2 * time.Minute)) - for _, v := range vs { - if v.Resource == "nzbget.downloads" && v.State != link.StateHealthy { - t.Errorf("an access whose group was given write is stated %+v", v) - } - } -} - -// A path that does not answer — a hard network mount whose server is down — is stated unhealthy within the bound, -// and stops neither this look nor the next: one blocked read per path, never one per look. -func TestAnAccessWhosePathDoesNotAnswerIsStatedWithinTheBound(t *testing.T) { - hung, library := t.TempDir(), t.TempDir() - release, finished := make(chan struct{}), make(chan struct{}) - var started atomic.Int32 - was, wasBound := apply.GroupOfPath, lookBound - apply.GroupOfPath = func(path string) (apply.PathGroup, error) { - if path == hung { - started.Add(1) - <-release - defer close(finished) - } - return apply.PathGroup{GID: 1001, Name: "media", Writable: true}, nil - } - lookBound = 50 * time.Millisecond - wasRoot := apply.RootGroupIDs - apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} } - t.Cleanup(func() { apply.GroupOfPath, lookBound, apply.RootGroupIDs = was, wasBound, wasRoot }) - // Run first: the read that hangs is let go and has returned before anything it read is put back. - t.Cleanup(func() { close(release); <-finished }) - - d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ - {"id":"nzbget.downloads","type":"access","path":"` + hung + `","mode":"read-write","grant":{}}, - {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{}}]}`)) - if err != nil { - t.Fatal(err) - } - j := newAccessJudge() - j.set(d, nil) - for look := 0; look < 3; look++ { - done := make(chan []link.ResourceHealth, 1) - go func() { vs, _ := j.look(time.Now()); done <- vs }() - select { - case vs := <-done: - for _, v := range vs { - switch v.Resource { - case "nzbget.downloads": - if v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "does not answer") { - t.Errorf("look %d: a path that does not answer is stated %+v", look, v) - } - case "importer.music": - if v.State != link.StateHealthy { - t.Errorf("look %d: the path that answers is stated %+v", look, v) - } - } - } - case <-time.After(5 * time.Second): - t.Fatalf("look %d did not return while one path hangs", look) - } - } - if n := started.Load(); n != 1 { - t.Errorf("%d reads were started on the path that hangs; want one", n) - } -} diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go new file mode 100644 index 0000000..d435a77 --- /dev/null +++ b/cmd/mesh-host/kept.go @@ -0,0 +1,431 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// An access is judged by what each of its users may do (novox/hq ADR 0325 §7, §10, §12). +// +// On every look the engine judges each module's access as its module's resource of kind `access`: waiting, with a +// wait of kind access, while the mesh may not keep its folder yet or its disk is not there; unhealthy for root's +// data on its placed filesystem, and for a kept folder that gives a user less than its right — read by entry and +// mask together, on the folder and the sweep's sample under it, which the sweep puts back where it can; healthy +// otherwise. The folder is the target, never in a reason nor in a wait's words: the controller says them on the +// operator's channel, which withholds a path (ADR 0234 §6). + +// judgedAccess is one access as the judge reads it. +type judgedAccess struct { + module, id, name, path string + taken bool + users []store.KeptUser + // leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries). + leftover bool +} + +// accessJudge reads every access on every look. Safe for the apply and the looking loop at once. +type accessJudge struct { + mu sync.Mutex + accesses []judgedAccess + // was is each access's last verdict, with since when and how many looks in a row found it unhealthy. + was map[string]link.ResourceHealth + + // reading guards inFlight: every folder whose look has not returned yet. + reading sync.Mutex + inFlight map[string]bool +} + +// accesses is the one access judge on this machine. +var accesses = newAccessJudge() + +func newAccessJudge() *accessJudge { + return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}} +} + +// set is what the declaration just applied asks: every access of a module, not held as found, with the users of +// its folder as every access of that folder gives them. An access no longer declared is forgotten. +func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) { + var as []judgedAccess + if d != nil { + users := map[string]map[uint32]store.KeptUser{} + taken := map[string]bool{} + for _, r := range d.Resources { + if a, ok := r.(*declaration.Access); ok { + path := filepath.Clean(a.Path) + if users[path] == nil { + users[path] = map[uint32]store.KeptUser{} + } + taken[path] = taken[path] || a.Taken + for _, u := range a.Users { + had, seen := users[path][uint32(u.UID)] + if !seen || (u.Use == declaration.AccessReadWrite && had.Use != declaration.AccessReadWrite) { + users[path][uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), + Use: u.Use, From: u.From} + } + } + } + } + for _, r := range d.Resources { + a, ok := r.(*declaration.Access) + if !ok || held[a.ID] || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { + continue + } + // An older controller's access is judged by nothing, as before ADR 0325: no state is said of it. + if !apply.Keeps(d, a.Path) { + continue + } + at := strings.LastIndex(a.ID, ".") + if at <= 0 { + continue + } + path := filepath.Clean(a.Path) + ja := judgedAccess{module: a.ID[:at], id: a.ID, name: a.ID[at+1:], path: path, taken: taken[path]} + for _, u := range users[path] { + ja.users = append(ja.users, u) + } + sort.Slice(ja.users, func(x, y int) bool { return ja.users[x].UID < ja.users[y].UID }) + as = append(as, ja) + } + } + sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id }) + j.mu.Lock() + defer j.mu.Unlock() + j.accesses = as + declared := map[string]bool{} + for _, a := range as { + declared[a.id] = true + } + for id := range j.was { + if !declared[id] { + delete(j.was, id) + } + } +} + +// lookBound is how long a look waits for one accessed folder to answer. A hard network mount whose server is down +// blocks a stat for ever, and one access must never stop every health statement this machine makes. +var lookBound = 5 * time.Second + +// verdict is one access's judgement in a look. +type verdict struct { + state, reason string + waits []link.AccessWait + act *link.AccessActRecord + entries []link.KeptEntry +} + +// judgeBounded judges one access within lookBound, outside any lock. A look that has not answered is said so; +// while it still hangs, later looks start no other on the same folder, so a dead mount costs one blocked read. +func (j *accessJudge) judgeBounded(a judgedAccess) (verdict, bool) { + j.reading.Lock() + if j.inFlight[a.path] { + j.reading.Unlock() + return verdict{}, false + } + j.inFlight[a.path] = true + j.reading.Unlock() + done := make(chan verdict, 1) + go func() { + v := judgeAccess(a) + j.reading.Lock() + delete(j.inFlight, a.path) + j.reading.Unlock() + done <- v + }() + select { + case v := <-done: + return v, true + case <-time.After(lookBound): + return verdict{}, false + } +} + +// judgeAccess is one access's judgement now (ADR 0325 §7, §10, §12), the sweep of its kept folder included. +func judgeAccess(a judgedAccess) verdict { + v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0) + switch v.Kind { + case apply.VerdictWaitKeep: + return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " + + "it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)", + waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, + What: apply.KeepWords(a.name), Placed: v.Placed}}} + case apply.VerdictWaitDisk: + return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)", + waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, + What: apply.DiskWords(a.name)}}} + case apply.VerdictRoot: + return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)} + case apply.VerdictUnknown: + // Not a fault and not the operator's: said as not known, and judged again on the next look. + return verdict{state: link.StateUnknown, reason: v.Why + "; judged again on the next look"} + case apply.VerdictBroken: + // Its words carry no path (AccessVerdict says them so): the reason reaches the operator's channel. + return verdict{state: link.StateUnhealthy, reason: v.Why} + case apply.VerdictMake: + return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " + + "its data lives (novox/hq ADR 0325 §9)"} + case apply.VerdictNobody: + return verdict{state: link.StateHealthy} + } + out := verdict{state: link.StateHealthy, entries: entriesOf(a.users)} + rec, _ := keptNow(a.path) + if act := rec.Act; act != nil { + out.act = &link.AccessActRecord{Act: act.Act, Access: a.id, Warrant: act.Warrant, By: act.By, State: act.State, + Started: act.Started, Ended: act.Ended, Folders: act.Folders, Files: act.Files, Kept: act.Kept, + Links: act.Links, Other: act.Other, Mounts: act.Mounts, Why: act.Why} + } + if !rec.Kept { + out.state, out.reason = link.StateStarting, "the controller says it was taken; the next apply keeps it" + return out + } + look := apply.LookKept(a.path, a.users, apply.SweepLimits) + var faults []string + if look.Failed != "" { + faults = append(faults, "the sweep could not look: "+apply.Pathless(errors.New(look.Failed))) + } + if len(look.Short) > 0 { + faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+ + "apply") + } + if n := look.Sweep.Unrepaired + look.Sweep.LastUnrepaired; n > 0 { + faults = append(faults, fmt.Sprintf("%d entries under it give a user less than its right and cannot be "+ + "put back: an owner's own bits, which the mesh never changes, or an entry the filesystem refused", n)) + } + if len(faults) > 0 { + out.state, out.reason = link.StateUnhealthy, strings.Join(faults, "; ") + return out + } + switch { + case rec.Act != nil && rec.Act.State == link.ActRunning: + out.state, out.reason = link.StateStarting, "the take act is adding its users' entries under it" + case !sameUsersList(rec.Users, rec.Want): + out.state, out.reason = link.StateStarting, "the entries under it are being changed to its users now" + } + return out +} + +// entriesOf is a kept folder's listing (ADR 0325 §4): each user, and beside the tool runner's what it stands for. +func entriesOf(users []store.KeptUser) []link.KeptEntry { + var out []link.KeptEntry + for _, u := range users { + e := link.KeptEntry{Identity: u.Identity, UID: int(u.UID), Use: u.Use, From: u.From} + if u.From == declaration.FromToolRunner { + e.Note = "the tool runner's account stands for every module whose tools use this folder, at the widest " + + "use any of them declares: a module that declared only read may write through it (novox/hq ADR " + + "0325 §4, until issue 569)" + } + out = append(out, e) + } + return out +} + +func sameUsersList(a, b []store.KeptUser) bool { + if len(a) != len(b) { + return false + } + seen := map[uint32]string{} + for _, u := range a { + seen[u.UID] = u.Use + } + for _, u := range b { + if use, ok := seen[u.UID]; !ok || use != u.Use { + return false + } + } + return true +} + +// keptNow is the record of one kept folder, as the engine's file holds it. +func keptNow(path string) (store.KeptFolder, bool) { + all, err := apply.KeptFolders.All() + if err != nil { + return store.KeptFolder{}, false + } + f, had := all[path] + return f, had +} + +// look reads every access now. Whether any verdict changed since the last look, for the loop to say it. +func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { + j.mu.Lock() + as := append([]judgedAccess(nil), j.accesses...) + j.mu.Unlock() + + read := make([]verdict, len(as)) + for i, a := range as { + v, answered := j.judgeBounded(a) + if !answered { + v = verdict{state: link.StateUnhealthy, reason: "its folder does not answer: judging it did not return " + + "within " + lookBound.String()} + } + read[i] = v + } + + // And the entries nobody wants that could not be taken out yet, each under the access that last named its + // folder, until its drop walk has run (ADR 0325 §8). + for _, l := range apply.Leftovers() { + as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path, + leftover: true}) + read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why}) + } + + j.mu.Lock() + defer j.mu.Unlock() + changed := false + out := make([]link.ResourceHealth, 0, len(as)) + for i, a := range as { + r := read[i] + kind := link.KindAccess + if a.leftover { + kind = link.KindKeptEntries + } + v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path, + State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries} + if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason { + v.Since = was.Since + v.Streak = was.Streak + } else { + changed = true + } + if r.state == link.StateUnhealthy { + v.Streak++ + } + j.was[a.id] = v + out = append(out, v) + } + return out, changed +} + +// withAccesses adds to a statement every access, as its module's resource of kind access. +func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health { + h.Resources = append(h.Resources, vs...) + return h +} + +// accessesHealthy says no access in vs is anything but healthy. +func accessesHealthy(vs []link.ResourceHealth) bool { + for _, v := range vs { + if v.State != link.StateHealthy { + return false + } + } + return true +} + +// keeper runs the walks under kept folders beside the applies, one at a time (ADR 0325 §8): poked after every +// apply and every look, it walks whatever folder's entries under it are not what was declared. +type keeper struct { + running, following atomic.Bool + say func(string) + watch *apply.Watcher +} + +// keeping is the one keeper on this machine. +var keeping = &keeper{say: func(string) {}} + +// poke starts a reconcile of the kept folders unless one is running, and has the watch follow what is kept. +func (k *keeper) poke(ctx context.Context) { + // The watch is set beside, never in the caller's way: watching a new folder's tree reads every folder of it. + if k.following.CompareAndSwap(false, true) { + go func() { + defer k.following.Store(false) + k.follow() + }() + } + if !k.running.CompareAndSwap(false, true) { + return + } + go func() { + defer k.running.Store(false) + for _, line := range apply.ReconcileKept(ctx) { + k.say(line) + } + }() +} + +// follow has the watch watch every kept folder with its users. +func (k *keeper) follow() { + if k.watch == nil { + return + } + all, err := apply.KeptFolders.All() + if err != nil { + return + } + folders := map[string][]store.KeptUser{} + for path, f := range all { + if f.Kept && len(f.Want) > 0 { + folders[path] = f.Want + } + } + k.watch.Set(folders) +} + +// startKeeping opens the watch and has the keeper say what it does. A machine without the watch is said so, and +// its sweep alone finds what is made. +func startKeeping(ctx context.Context, say func(string)) { + keeping.say = say + w, err := apply.NewWatcher() + if err != nil { + say(fmt.Sprintf("the kept folders are not watched (%v): the sweep alone finds what is made in them", err)) + } else { + keeping.watch = w + go w.Run(ctx) + } + keeping.poke(ctx) +} + +// accessActFrom is the take act as the controller asks it over the link (ADR 0325 §8): judged against the +// declaration this node keeps, and run beside the applies. +func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc { + return func(act link.AccessAct) (string, error) { + kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + if errors.Is(err, store.ErrNothingDeclared) { + return "", errors.New("this node keeps no declaration, so it is declared no access") + } + if err != nil { + return "", err + } + d, err := declaration.Parse(kept.Declaration) + if err != nil { + return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + _, said, err := apply.TakeFolder(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By, + apply.ExecRunner) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err) + return "", err + } + return said, nil + } +} + +// withWaiting is held with every id of waiting beside it: what liveness does not judge. +func withWaiting(held, waiting map[string]bool) map[string]bool { + if len(waiting) == 0 { + return held + } + out := map[string]bool{} + for id := range held { + out[id] = true + } + for id := range waiting { + out[id] = true + } + return out +} diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go new file mode 100644 index 0000000..209ce37 --- /dev/null +++ b/cmd/mesh-host/kept_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// Row 4: the listing of a kept folder says, beside the tool runner's entry, that it stands for every module whose +// tools use the folder at the widest use (novox/hq ADR 0325 §4). +func TestTheListingSaysWhatTheToolRunnersEntryStandsFor(t *testing.T) { + entries := entriesOf([]store.KeptUser{ + {Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: declaration.FromContainer}, + {Identity: "nox", UID: 61003, Use: "read-write", From: declaration.FromToolRunner}, + }) + if entries[0].Note != "" || !strings.Contains(entries[1].Note, "widest use") { + t.Fatalf("the listing: %+v", entries) + } +} + +// Row 7, the engine's part: a folder that holds data and was never taken is said waiting with one wait of kind +// access — the access by its module's own id, the part its module, and words with no path (novox/hq ADR 0325 §7). +func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { + dir := t.TempDir() + lib := filepath.Join(dir, "movies") + if err := os.MkdirAll(filepath.Join(lib, "a film"), 0o755); err != nil { + t.Fatal(err) + } + // Run as root (the build seat's merge-check), the test's folders stand for an operator's, never root's. + if os.Geteuid() == 0 { + for _, p := range []string{dir, lib, filepath.Join(lib, "a film")} { + if err := os.Lchown(p, 65534, 65534); err != nil { + t.Fatal(err) + } + } + } + v := judgeAccess(judgedAccess{module: "radarr", id: "radarr.movies", name: "movies", path: lib, + users: []store.KeptUser{{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: "container"}}}) + if v.state != link.StateWaiting || len(v.waits) != 1 { + t.Fatalf("not waiting with one wait: %+v", v) + } + raw, _ := json.Marshal(v.waits[0]) + want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` + + `"placed":"` + v.waits[0].Placed + `"}` + if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") { + t.Fatalf("the wait reads %s", raw) + } + if strings.Contains(v.reason, dir) { + t.Fatalf("the reason names the path: %s", v.reason) + } +} + +// An older controller's accesses (no users, no identities) are judged by nothing, as before ADR 0325: the statement +// carries no resource for them, waiting, unknown or kept-entries. +func TestAnOlderControllersAccessesSayNothing(t *testing.T) { + dir := t.TempDir() + d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"radarr.library","type":"access","path":"` + filepath.Join(dir, "library") + `","mode":"read-write"}, + {"id":"plex.movies","type":"access","path":"` + filepath.Join(dir, "absent") + `"}]}`)) + if err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(dir, "library"), 0o755); err != nil { + t.Fatal(err) + } + j := newAccessJudge() + j.set(d, nil) + if vs, _ := j.look(time.Now()); len(vs) != 0 { + t.Fatalf("an older controller's accesses are judged: %+v", vs) + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index feb1da7..a1d21ab 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -241,6 +241,8 @@ func run(ctx context.Context, command string, opts options) error { // found (novox/hq issue 339). handedOverFrom(opts.state) foundState.Store(opts.state) + // And what it knows of the shared folders it keeps is beside its state (novox/hq ADR 0325 §5). + apply.KeptFolders = store.NewKeptFile(opts.state) // And the guard asks the running container runtimes where they keep their data (novox/hq issue 339). apply.AskRuntimes = apply.ExecRunner switch command { @@ -1121,6 +1123,8 @@ func runLink(ctx context.Context, opts options) error { // this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224). sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state))) go sched.Run(ctx) + // The watch on every kept folder and the walks under them, beside the applies (novox/hq ADR 0325 §6, §8). + startKeeping(ctx, func(line string) { say(line) }) // The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every // report this host makes carries what it says. @@ -1232,8 +1236,10 @@ func runLink(ctx context.Context, opts options) error { // A read-only judgement of the directories this node is declared, asked by the controller (novox/hq // issue 358): which the walk that follows no link another account could make would refuse. JudgeDirectories: judgeDirectoriesFrom(opts.state), - Say: say, - Timeout: opts.timeout, + // The take act on an access's folder, on the operator's warrant (novox/hq ADR 0325 §8). + AccessAct: accessActFrom(opts.state, opts.timeout), + Say: say, + Timeout: opts.timeout, } worker := make(chan struct{}) go func() { @@ -1596,7 +1602,8 @@ const ( // judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 -// rule 6). +// rule 6) — but for the sweep of a kept folder, which ADR 0325 §10 has every look make: it puts back an entry or a +// mask an app narrowed, and nothing else. func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() @@ -1635,8 +1642,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - // Every read-write access that grants, read on every look (novox/hq ADR 0312 §2). + // Every access, judged on every look, its kept folder swept (novox/hq ADR 0325 §10); and a walk stopped + // at its bound goes on. accessSt, accessesChanged := accesses.look(time.Now()) + keeping.poke(ctx) owed = owed || accessesChanged if accessesChanged { for _, v := range accessSt { @@ -1691,10 +1700,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa // healthAsReported is a statement as the report and the event carry it. func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { - // GrantContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a - // user's declared `root` (novox/hq ADR 0266), and grants a read-write access's path group (novox/hq ADR 0312), - // which is what tells the controller it may be sent any of those fields. - h := &link.Health{Contract: link.GrantContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} + // IdentityContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a + // user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, a container's identity and + // an access's users (novox/hq ADR 0325), which is what tells the controller it may be sent any of those fields. + h := &link.Health{Contract: link.IdentityContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, @@ -2075,7 +2084,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just // applied, so every resource this apply started says `starting` and the gate waits for it. if j := judging; j != nil { - j.Set(liveness.LongRunning(declared, held)) + // A container that waits for an accessed folder not there was not created: the access says why, and + // liveness judges it neither alive nor dead (novox/hq ADR 0325 §9). + j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnAccess(declared)))) st, _ := j.Look(ctx) netJudge.declare(declaredResolvConf(declared)) var netSt *network.Statement @@ -2093,13 +2104,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // said with it, relogin needed included, in the report that says the apply. var accountSt *accounts.Statement if a := accountJudge; a != nil { - // A read-write access's account too (novox/hq ADR 0312 §6): relogin needed is said for a session of - // the account's own manager that began before the grant, as for a user resource's group. - a.Set(append(accounts.Of(declared, held), apply.GrantedAccounts(declared, held)...)) + a.Set(accounts.Of(declared, held)) as, _ := a.Look(ctx) accountSt = &as } accesses.set(declared, held) + // What the apply changed in who uses a kept folder is walked under it now, beside (ADR 0325 §8). + keeping.poke(ctx) accessSt, _ := accesses.look(time.Now()) report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), foundDirectories(updated)), accessSt) diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index f8cddc2..2036bce 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -87,8 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at}, }} h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as) - // At RootContract or later: an engine that also grants an access's group says GrantContract (novox/hq ADR 0312). - if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.GrantContract { + // At RootContract or later: an engine that also reads identities and keeps shared folders says IdentityContract + // (novox/hq ADR 0325), one above ADR 0312's GrantContract. + if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.IdentityContract || + link.IdentityContract != link.GrantContract+1 { t.Fatalf("contract %d", h.Contract) } got := map[string]string{} diff --git a/internal/apply/access_test.go b/internal/apply/access_test.go index f4f6187..f045bb5 100644 --- a/internal/apply/access_test.go +++ b/internal/apply/access_test.go @@ -3,7 +3,6 @@ package apply import ( "context" "os" - "strings" "testing" "github.com/novox/mesh-host/internal/store" @@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) { } } -// Absent is refused, not created. A bind mount whose source does not exist is made by the -// container runtime as root, with whatever mode it picks — the silent wrong-ownership -// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather -// than conjuring a directory it does not own. -func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) { - missing := t.TempDir() + "/media/library" // named, never created - d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`) - - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, - store.OriginDeclared, noServices, nil, nil) - if err == nil { - t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime") - } - if !strings.Contains(err.Error(), "the operator must provide") || - !strings.Contains(err.Error(), "does not own") { - t.Fatalf("the refusal does not say whose the path is: %v", err) - } - if _, statErr := os.Stat(missing); statErr == nil { - t.Fatal("the host created the path it does not own") - } -} - // Undeclaring an access never removes the path. Unassigning the module that reached the media // library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a // directory the mesh never made. The record is dropped; the operator's data is left exactly as it diff --git a/internal/apply/acl.go b/internal/apply/acl.go new file mode 100644 index 0000000..2b8e8c5 --- /dev/null +++ b/internal/apply/acl.go @@ -0,0 +1,242 @@ +package apply + +// The POSIX ACL a kept folder carries, as the kernel stores it (novox/hq ADR 0325 §5, issue 567). +// +// **Read and written as the kernel's own extended attribute, never through setfacl.** Linux keeps a file's access +// ACL in the attribute system.posix_acl_access and a folder's default ACL in system.posix_acl_default, each a +// little-endian header (version 2) followed by entries of a tag, a permission and an id. The engine reads and +// writes them on a descriptor it opened itself (fgetxattr, fsetxattr), so a name swapped for a link between the +// look and the change changes nothing, and no program is run per file of a library of millions. +// +// **What the mesh changes is narrow.** It adds or sets one named-user entry per uid it keeps, widens the mask to +// what the entries need, and on a file with no ACL yet writes the owning group's entry as exactly the group bits +// the mode had: owner, owning group, the owner's bits, others' bits and every other entry stay as found. + +import ( + "encoding/binary" + "errors" + "fmt" + "sort" +) + +// The attributes and the tags of the kernel's ACL format (linux/posix_acl_xattr.h). +const ( + aclAccessXattr = "system.posix_acl_access" + aclDefaultXattr = "system.posix_acl_default" + aclVersion = 2 + + aclUserObj = 0x01 + aclUser = 0x02 + aclGroupObj = 0x04 + aclGroup = 0x08 + aclMask = 0x10 + aclOther = 0x20 + + // aclUndefinedID is the id of an entry that names nobody: the owner, the owning group, the mask, others. + aclUndefinedID = 0xFFFFFFFF +) + +// The permission bits of one entry. +const ( + permRead = 4 + permWrite = 2 + permExec = 1 +) + +// aclEntry is one entry: its tag, its permission bits (rwx as 4, 2, 1) and, for a named user or group, its id. +type aclEntry struct { + Tag uint16 + Perm uint16 + ID uint32 +} + +// acl is a whole ACL as the kernel holds it. +type acl []aclEntry + +// decodeACL reads an attribute's bytes. An attribute that is not version 2, or not whole entries, is refused. +func decodeACL(raw []byte) (acl, error) { + if len(raw) < 4 || (len(raw)-4)%8 != 0 { + return nil, fmt.Errorf("an ACL attribute of %d bytes is not a header and whole entries", len(raw)) + } + if v := binary.LittleEndian.Uint32(raw); v != aclVersion { + return nil, fmt.Errorf("an ACL attribute of version %d, and the engine reads version %d", v, aclVersion) + } + out := make(acl, 0, (len(raw)-4)/8) + for at := 4; at < len(raw); at += 8 { + out = append(out, aclEntry{ + Tag: binary.LittleEndian.Uint16(raw[at:]), + Perm: binary.LittleEndian.Uint16(raw[at+2:]), + ID: binary.LittleEndian.Uint32(raw[at+4:]), + }) + } + return out, nil +} + +// encode is the attribute's bytes, the entries in the order the kernel requires: by tag, then by id. +func (a acl) encode() []byte { + sorted := append(acl(nil), a...) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].Tag != sorted[j].Tag { + return sorted[i].Tag < sorted[j].Tag + } + return sorted[i].ID < sorted[j].ID + }) + out := make([]byte, 4, 4+8*len(sorted)) + binary.LittleEndian.PutUint32(out, aclVersion) + for _, e := range sorted { + var b [8]byte + binary.LittleEndian.PutUint16(b[0:], e.Tag) + binary.LittleEndian.PutUint16(b[2:], e.Perm) + binary.LittleEndian.PutUint32(b[4:], e.ID) + out = append(out, b[:]...) + } + return out +} + +// minimalACL is the ACL a mode alone stands for: the owner's, the owning group's and others' bits. +func minimalACL(mode uint32) acl { + return acl{ + {Tag: aclUserObj, Perm: uint16(mode>>6) & 7, ID: aclUndefinedID}, + {Tag: aclGroupObj, Perm: uint16(mode>>3) & 7, ID: aclUndefinedID}, + {Tag: aclOther, Perm: uint16(mode) & 7, ID: aclUndefinedID}, + } +} + +// find is the index of the entry with tag and id, or -1. +func (a acl) find(tag uint16, id uint32) int { + for i, e := range a { + if e.Tag == tag && (e.ID == id || (tag != aclUser && tag != aclGroup)) { + return i + } + } + return -1 +} + +// perm is the bits of the entry with tag and id, and whether there is one. +func (a acl) perm(tag uint16, id uint32) (uint16, bool) { + if i := a.find(tag, id); i >= 0 { + return a[i].Perm, true + } + return 0, false +} + +// check refuses an ACL the kernel would: one owner's, owning group's and others' entry each, and a mask whenever +// a named entry is there. +func (a acl) check() error { + count := map[uint16]int{} + named := false + for _, e := range a { + count[e.Tag]++ + if e.Tag == aclUser || e.Tag == aclGroup { + named = true + } + } + if count[aclUserObj] != 1 || count[aclGroupObj] != 1 || count[aclOther] != 1 || count[aclMask] > 1 { + return errors.New("an ACL without exactly one owner's, owning group's and others' entry") + } + if named && count[aclMask] != 1 { + return errors.New("an ACL with named entries and no mask") + } + return nil +} + +// keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask +// widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps +// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning +// group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. +// Nothing else is changed. +func keptACL(a acl, need map[uint32]uint16) acl { + out := append(acl(nil), a...) + var widen uint16 + for uid, bits := range need { + widen |= bits + if i := out.find(aclUser, uid); i >= 0 { + out[i].Perm = bits + } else { + out = append(out, aclEntry{Tag: aclUser, Perm: bits, ID: uid}) + } + } + if i := out.find(aclMask, 0); i >= 0 { + old := out[i].Perm + out[i].Perm |= widen + if added := out[i].Perm &^ old; added != 0 { + // **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does + // not set here were limited by the old mask: a file made at 0644 under a default ACL carries the + // owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask + // now adds cannot keep its effective right without losing that bit: exactly those bits are taken off + // it, and nothing else of it changes. An entry holding none of them is not touched at all. + for j := range out { + e := &out[j] + _, ours := need[e.ID] + if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 { + e.Perm &^= added + } + } + } + return out + } + if len(need) == 0 { + return out + } + group, _ := out.perm(aclGroupObj, 0) + return append(out, aclEntry{Tag: aclMask, Perm: group | widen, ID: aclUndefinedID}) +} + +// withoutUsers is a with the named-user entry of each uid of drop taken out, and nothing else changed: the mask +// stays as it was, so no other entry loses a right. +func withoutUsers(a acl, drop map[uint32]bool) acl { + out := make(acl, 0, len(a)) + for _, e := range a { + if e.Tag == aclUser && drop[e.ID] { + continue + } + out = append(out, e) + } + return out +} + +// effective is the right a uid has through a, given the file's owner and mode: the owner's bits for the owner, +// else its named entry limited by the mask, else nothing a named entry gives (the owning group's and others' +// rights are not the mesh's to count on). An ACL with no mask limits nothing. +func effective(a acl, uid, owner uint32, mode uint32) uint16 { + if uid == owner { + return uint16(mode>>6) & 7 + } + bits, ok := a.perm(aclUser, uid) + if !ok { + return 0 + } + if mask, has := a.perm(aclMask, 0); has { + bits &= mask + } + return bits +} + +// needOn is what a use needs on a folder (read and enter; with read-write, write too) or on a file (read; with +// read-write, write): entering means nothing on a file, and an execute bit given to data would be a right nobody +// asked for. +func needOn(readWrite, folder bool) uint16 { + bits := uint16(permRead) + if readWrite { + bits |= permWrite + } + if folder { + bits |= permExec + } + return bits +} + +// permWords is bits as ls writes them: "rwx", "r-x". +func permWords(bits uint16) string { + out := []byte("---") + if bits&permRead != 0 { + out[0] = 'r' + } + if bits&permWrite != 0 { + out[1] = 'w' + } + if bits&permExec != 0 { + out[2] = 'x' + } + return string(out) +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1ff9a5b..0934dcd 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -262,6 +262,12 @@ func ApplyMindingWindows( } report = Report{} + // **No module identity is root** (novox/hq ADR 0325 §11): an identity or a container told a root number refuses + // the declaration whole, before anything of it is made or started. + if err := refuseRootIdentities(d); err != nil { + return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report} + } + declared := map[string]bool{} for _, r := range d.Resources { declared[r.Identity()] = true @@ -315,7 +321,7 @@ func ApplyMindingWindows( // (novox/hq ADR 0252), on the unit's rule above: two modules may each want the account in one group, // and the one going takes nothing the other still needs. wanted := groupsWanted(d.Resources) - // And what a record going hands to a resource that still asks for its group (novox/hq ADR 0312 §5). + // And what a record going hands to a resource that still asks for its group (novox/hq ADR 0252 §3). handed := Handed{} removeOrphan := func(orphan store.Applied) error { @@ -343,8 +349,25 @@ func ApplyMindingWindows( case declaration.TypeUser: action, detail, err = removeUser(ctx, sys, orphan, run, wanted, handed) case declaration.TypeAccess: - // The path is left as it is; what it granted an account is taken back (novox/hq ADR 0312 §5). - action, detail = removeAccess(ctx, sys, orphan, run, wanted, handed) + // The folder is left as it is: what the mesh added to it is taken out by the walk once no access names + // it (novox/hq ADR 0325 §8), never by removing the access's record. + action, detail = "forgotten", "an operator-owned path is never the host's to remove" + if _, kept := keptRecord(filepath.Clean(orphan.Target)); kept { + detail += "; the entries the mesh added go once no access names it" + } + gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed) + if said != "" { + detail = joinDetail(detail, said) + if gave { + action = "restored" + } + } + if len(left) > 0 { + // Kept recorded with what is still to take back, and tried again (ADR 0177's waiting removal). + orphan.Groups = left + known.Record(orphan) + err = fmt.Errorf("%w: %s", errRemovalWaits, said) + } default: action, detail, err = remove(ctx, sys, orphan, run, made) } @@ -516,12 +539,11 @@ func ApplyMindingWindows( // which declared directories were already here: one this apply makes is the mesh's, one found may not be. refreshRuntimeRoots(ctx) in.refused = refusedPlaces(d.Resources) - // The group of every read-write access that grants, read once (novox/hq ADR 0312): what each container is - // created with, and what each account is asked to be in beside its user resources' groups — read before any - // orphan goes, so one access going leaves an account in a group another still grants. - in.grants = grantsOf(d.Resources, in.refused) - in.containerGroups = containerGroups(d.Resources, in.grants) - wanted.withGrants(d.Resources, in.grants) + // What every declared access asks of its folder, several accesses of one folder as one (novox/hq ADR 0325 §5), + // and every accessed folder, which no container mounts while it is not there (§9). + in.folders = foldersOf(d.Resources, in.refused) + in.identities = len(d.Identities) > 0 + in.accesses = accessPaths(d.Resources, in.refused, in.identities) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} for _, resource := range d.Resources { @@ -563,6 +585,34 @@ func ApplyMindingWindows( sort.Strings(out) return out } + // **Every module's identity is made before anything of it is applied** (novox/hq ADR 0325 §3): a directory + // it owns is given to it, and a container is told to run as it. One that cannot be made — its name or numbers + // another's here — fails, naming whose, and its module is not attempted; the rest of the machine is. + var identityFailures []*Error + if len(d.Identities) > 0 { + tool, toolErr := toolFor(sys) + for _, id := range d.Identities { + var outcome Outcome + err := toolErr + if err == nil { + outcome, err = applyIdentity(ctx, tool, id, &known, run) + } + if err != nil { + // Counted with the others once the resources are through: here, before the guard, it would read + // as the guard not being up. + identityFailures = append(identityFailures, &Error{Resource: "identity " + id.Name, Err: err}) + log(fmt.Sprintf(" failed identity %s: %v", id.Name, err)) + gated[id.Module] = true + gatedWhy[id.Module] = "its identity " + id.Name + " could not be made, so nothing of this module " + + "was attempted" + continue + } + report.Outcomes = append(report.Outcomes, outcome) + if outcome.Action != "unchanged" { + log(fmt.Sprintf(" %s %s: %s", outcome.Action, outcome.ID, outcome.Detail)) + } + } + } for i, resource := range ordered { if !orphansRemoved && i == guardFirst { // **Only a guard that is up may let the filter go.** Removing the derived filter's @@ -925,6 +975,8 @@ func ApplyMindingWindows( } } + failures = append(failures, identityFailures...) + // A replacement that did not apply — failed, skipped behind its module's step, held — leaves // what it replaces running and recorded, said, for the next apply to hand over. unhanded := make([]string, 0, len(handover)) @@ -945,6 +997,19 @@ func ApplyMindingWindows( } } + // Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder. + if origin == store.OriginDeclared { + // A folder no access names any more has what the mesh added taken out by the next walk (ADR 0325 §8). + forgetUndeclared(in.folders) + // And an identity no longer declared goes, once what ran as it went above (ADR 0325 §3). + if tool, err := toolFor(sys); err == nil { + for _, o := range removeIdentities(ctx, tool, d.Identities, &known, run) { + report.Outcomes = append(report.Outcomes, o) + log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail)) + } + } + } + // A converged node whose found firewall was in force retires it only now, once everything — // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100), and on every // converged apply, not once (ADR 0168). Skipped, it is said: a step that does nothing is never @@ -1072,6 +1137,14 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru if err := refuseMounts(res, in.refused); err != nil { return begin(r), err } + // Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9). + if id := mountsMissing(ctx, res, in.accesses); id != "" { + out := begin(r) + out.Action = "waiting" + out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " + + "created once it is (novox/hq ADR 0325 §9)" + return out, nil + } } // Nothing below a home is touched through a link an account put there (novox/hq ADR 0266). // And nothing is unpacked or written into below an agent's home (agent_home_guard.go). @@ -1304,45 +1377,6 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O // library, a download spool that several modules share — and the host's only job is to be sure it // is there before anything mounts it. // -// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by -// the container runtime, as root, with whatever mode it picks — which is exactly the silent -// wrong-ownership 04-ISSUES/026 records. So the host checks first and says plainly that the -// operator must provide the path, rather than conjuring a directory it does not own and cannot -// give the right owner. Nothing is written, so this never reports a change: the machine did not -// move, the host merely confirmed a fact about it. -// -// **A read-write access grants its path's group** (novox/hq ADR 0312): read, never changed — grantAccess. -func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, - in inputs) (Outcome, error) { - out := begin(r) - info, err := os.Stat(r.Path) - if errors.Is(err, os.ErrNotExist) { - return out, fmt.Errorf( - "%s is not there, and the mesh does not own it — the operator must provide it. It is "+ - "shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+ - "nothing, so a missing one is said here rather than made as root by the container "+ - "runtime", r.Path) - } - if err != nil { - return out, err - } - if !info.IsDir() { - return out, fmt.Errorf( - "%s is not a directory, and an access is a shared directory the operator provides", - r.Path) - } - mode := declaration.AccessRead - if r.Mode != "" { - mode = r.Mode - } - out.Action = "unchanged" - out.Detail = "operator-owned; present, " + mode + ", nothing managed" - if err := grantAccess(ctx, sys, r, run, previous, in, &out); err != nil { - return out, err - } - return out, nil -} - // keepFound, when not nil, is where the original of a file this host has no record of is kept // before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { @@ -2160,10 +2194,12 @@ type inputs struct { // dirsBefore is every declared directory that was on the machine when this apply began (novox/hq issue // 339): one the apply itself made — a file's parent — is the mesh's, one that was there may not be. dirsBefore map[string]bool - // grants is what every read-write access that grants grants, by access id, read once (novox/hq ADR 0312); - // containerGroups the number of every group granted to a container, by container id. - grants map[string]grant - containerGroups map[string][]int + // folders is what every declared access asks of its folder, by path (novox/hq ADR 0325 §5); accesses every + // accessed folder with its access's id, which no container mounts while it is not there (§9). + folders map[string]*folderWant + // identities is that the declaration carries identities: its controller speaks ADR 0325 for every access. + identities bool + accesses map[string]string // handed is every group a record that went this apply handed to a resource still asking for it. handed Handed } @@ -2240,15 +2276,13 @@ func (in inputs) reads(r *declaration.Container) map[string]string { // containerSpec is the identity of a declared container: everything that, if changed, means // the running container is no longer what was asked for. func containerSpec(r *declaration.Container, in inputs) string { - return containerSpecReading(r, in.declares, in.reads(r), in.containerGroups[r.ID]) + return containerSpecReading(r, in.declares, in.reads(r)) } // containerSpecReading is containerSpec with what the container reads already read — so an // applier that also records those digests reads each file once, and the label and the record // cannot disagree about a file that moved between two reads. -// -// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312). -func containerSpecReading(r *declaration.Container, declares, reads map[string]string, groups []int) string { +func containerSpecReading(r *declaration.Container, declares, reads map[string]string) string { keys := make([]string, 0, len(r.Env)) for k := range r.Env { keys = append(keys, k) @@ -2327,11 +2361,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s if h := r.Health; h.RunByRuntime() { b.WriteString("health " + strings.Join(runtimeCheckArgs(h), " ") + "\n") } - // And the groups its read-write accesses grant it (novox/hq ADR 0312 §6): a process's supplementary groups - // are fixed when the container is created, so one gained, dropped or renumbered is a container recreated - // with it. Only when there is one, so no container that is granted nothing has its digest moved. - for _, gid := range groups { - b.WriteString("group " + strconv.Itoa(gid) + "\n") + // And who it runs as (novox/hq ADR 0325 §2, §3): fixed when the container is created, so an identity told + // another way, or renumbered, is a container recreated with it. Only when there is one, so no container that + // runs as its image says has its digest moved. + if args := identityArgs(r); len(args) > 0 { + b.WriteString("identity " + strings.Join(args, " ") + "\n") } // **What this container reads is part of what it is.** // @@ -2433,8 +2467,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, out := begin(r) // Read once, so the spec and the record agree on what was read even if a file moves under them. reads := in.reads(r) - groups := in.containerGroups[r.ID] - want := containerSpecReading(r, in.declares, reads, groups) + want := containerSpecReading(r, in.declares, reads) out.reads = reads cri, err := containerRuntime(ctx, run) @@ -2462,7 +2495,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, } if r.RunOnce { - return applyRunOnce(ctx, r, run, cri, want, groups, previous) + return applyRunOnce(ctx, r, run, cri, want, previous) } before, err := containerState(ctx, r.Name, run) @@ -2507,7 +2540,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, // genuine recreate. The trade-off, stated: a container that was ALREADY stale when the host // upgraded — created against a file that has since changed — is not caught by this, and could // not be by the alternative either, which recreates it without knowing whether it needed to. - legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil, groups) && + legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil) && (wasReading == nil || sameReads(wasReading, reads)) // **A container a maintenance window holds still is left as it is** (novox/hq issue 224, ADR @@ -2577,7 +2610,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, for _, v := range r.Volumes { args = append(args, "--volume", v) } - args = append(args, groupAddArgs(groups)...) + args = append(args, identityArgs(r)...) // **The one check the container carries is the one the module declared** (ADR 0240 rule 3): an // exec command, or the image's own adopted by name, with the declared timing. Nothing else on the // machine sets a container's check. @@ -2634,9 +2667,9 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, } case len(reasons) > 0: out.Detail = "recreated to pick up " + strings.Join(reasons, ", ") - case len(groups) > 0 && before.Spec == containerSpecReading(r, in.declares, reads, nil): - // It was created before its read-write accesses granted it anything (novox/hq ADR 0312 §6). - out.Detail = "recreated to hold the groups its read-write accesses grant (gid " + gidWords(groups) + ")" + case r.RunAs != nil && before.Spec == specWithoutIdentity(r, in.declares, reads): + // It was created before it was told its module's identity (novox/hq ADR 0325 §3). + out.Detail = "recreated to run as its module's identity (novox/hq ADR 0325 §3)" default: out.Detail = "replaced; a container's configuration is fixed when it is created" } @@ -2676,7 +2709,7 @@ func runtimeCheckArgs(h *declaration.Health) []string { // - It writes a seed only if it is absent and never reconciles it, so what a running program // grows in that seed afterward is never wiped (04-ISSUES/035). The host's marker keeps the // step from re-running; the step's own code keeps it from clobbering on the pass it does run. -func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, groups []int, +func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, previous store.Applied) (Outcome, error) { out := begin(r) @@ -2696,7 +2729,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri // Run in the foreground so the runtime waits for the container and hands back its exit code. // No --detach and no --restart: a step that is restarted is not a step. - args := foregroundRunArgs(r, want, groups) + args := foregroundRunArgs(r, want) if _, err := run(ctx, cri, args...); err != nil { // A non-zero exit or a runtime that could not start it. Either way the step did not make @@ -2718,9 +2751,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri // exit code — no --detach, no --restart, because a step that is restarted is not a step. Shared by // a run-once step (novox/hq ADR 0052) and by one fire of a scheduled step (ADR 0053), which are the // same "run the container and let it exit" up to how often it happens. -// -// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312). -func foregroundRunArgs(r *declaration.Container, want string, groups []int) []string { +func foregroundRunArgs(r *declaration.Container, want string) []string { args := []string{"run", "--name", r.Name} for _, file := range r.EnvFile { args = append(args, "--env-file", file) @@ -2735,7 +2766,7 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st for _, v := range r.Volumes { args = append(args, "--volume", v) } - args = append(args, groupAddArgs(groups)...) + args = append(args, identityArgs(r)...) for _, h := range r.Hosts { args = append(args, "--add-host", h) } @@ -2744,6 +2775,31 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st return args } +// specWithoutIdentity is the spec r had before it was told who to run as. +func specWithoutIdentity(r *declaration.Container, declares, reads map[string]string) string { + plain := *r + plain.RunAs = nil + return containerSpecReading(&plain, declares, reads) +} + +// identityArgs is the runtime's arguments that run a container as its module's identity (novox/hq ADR 0325 §2): +// the image told by the variables it reads, or by the runtime's own user option, by number — a container's own +// account database is not the machine's. None for a container that runs as its image says. +func identityArgs(r *declaration.Container) []string { + id := r.RunAs + if id == nil { + return nil + } + if id.Mode == declaration.IdentityByRun { + return []string{"--user", strconv.Itoa(id.UID) + ":" + strconv.Itoa(id.GID)} + } + args := []string{"--env", id.UserEnv + "=" + strconv.Itoa(id.UID)} + if id.GroupEnv != "" { + args = append(args, "--env", id.GroupEnv+"="+strconv.Itoa(id.GID)) + } + return args +} + // ensureImage makes the pinned image present on the node without running anything. // // A service or a run-once container gets its image as a side effect of `docker run` — the first run diff --git a/internal/apply/grants.go b/internal/apply/grants.go deleted file mode 100644 index da702cf..0000000 --- a/internal/apply/grants.go +++ /dev/null @@ -1,514 +0,0 @@ -package apply - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/user" - "path/filepath" - "slices" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/novox/mesh-host/internal/accounts" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// A read-write access grants the path's own group (novox/hq ADR 0312, issue 537). -// -// **An access is not owning.** ADR 0051 left the operator's shared paths exactly as they are: the engine -// confirms one is there and mounts it, and sets no owner, group or mode on it or below it. Nothing decided how -// a `read-write` access then becomes writable, and a module whose identity was not the path's owner could not -// write it at all — a download client could not remove a folder it had filed (issue 535). -// -// **The path's group is the operator's statement of who may share it.** So the engine reads, at apply, which -// group owns the accessed path — only the machine knows — and makes the identities the controller names for -// the access members of it: each container that mounts it gains the group as a supplementary group of its -// process (`--group-add`, by number, because a container's own group database is not the machine's), and the -// account that runs the module's tools is put in it as an account group (ADR 0252 §1). The module then writes -// the path as its group may, and as nothing else. -// -// **The path is never changed** (§2). A group that cannot write the path is the operator's choice, as the data -// is: the access grants nothing, is stated unhealthy on every look, saying so and that the operator decides, and -// nothing is done to make it writable. **A read access grants nothing** (§3). **The root group is never granted** (§4): a -// path owned by group 0, or by a group whose members act as root (accounts.RootGroups, the account judge's own -// closed list, judged by the numbers those groups have here), or by a number the machine names nothing for, is -// refused by name. -// -// **What was granted is taken back as ADR 0252 takes back** (§5): the access records each group it put the -// account in that the account was not in before; when no declared resource asks for it any more, the account -// leaves it, and a group the account held before is never recorded and never taken back. A container needs no -// record: the group is part of what it is created as, so one no longer granted is recreated without it. -// -// **And what runs the code is restarted to hold it** (§6). A container is recreated with the group, because -// its spec carries it. The tool runner — a process of the machine's own service manager with `User=` — is -// restarted because the controller names the access under its `restart-on`, and the access is reported -// updated exactly when the account's groups moved: that manager reads the account's groups from the database -// each time it starts the unit, so the restarted process holds the group with no new login. A unit of the -// account's own manager, or a login session, keeps the groups its login began with: the account judge -// (internal/accounts) says *relogin needed* there, for the access's account as for a user resource's. - -// PathGroup is the group that owns an accessed path, as the engine reads it. -type PathGroup struct { - GID int - // Name is the group's name in the machine's group database; empty when the database names nothing for - // the number. - Name string - // Writable is whether the group's write bit is set on the path itself. - Writable bool -} - -// Words names the group for a person: its name and number, or its number alone. -func (g PathGroup) Words() string { - if g.Name == "" { - return fmt.Sprintf("gid %d", g.GID) - } - return fmt.Sprintf("%s (gid %d)", g.Name, g.GID) -} - -// GroupOfPath reads which group owns a path, by its number, its name and whether it may write. Reads only. A -// variable so a test can stand in for a machine whose groups it cannot make. -var GroupOfPath = func(path string) (PathGroup, error) { - info, err := os.Stat(path) - if err != nil { - return PathGroup{}, err - } - _, gid, ok := statOwner(info) - if !ok { - return PathGroup{}, fmt.Errorf("the group that owns %s cannot be read on this machine", path) - } - g := PathGroup{GID: gid, Writable: info.Mode().Perm()&0o020 != 0} - if found, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil { - g.Name = found.Name - } - return g, nil -} - -// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine -// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine -// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number -// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and -// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can -// stand in. -var RootGroupIDs = func() map[int]string { - rootIDs.Lock() - defer rootIDs.Unlock() - if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute { - rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now() - } - return rootIDs.ids -} - -var rootIDs struct { - sync.Mutex - ids map[int]string - at time.Time -} - -func readRootGroupIDs() map[int]string { - out := map[int]string{} - if raw, err := os.ReadFile("/etc/group"); err == nil { - for name, gid := range groupEntries(string(raw)) { - if slices.Contains(accounts.RootGroups, name) { - out[gid] = name - } - } - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - for _, name := range accounts.RootGroups { - raw, err := exec.CommandContext(ctx, "getent", "group", name).Output() - if err != nil { - continue - } - for found, gid := range groupEntries(string(raw)) { - if found == name { - out[gid] = name - } - } - } - return out -} - -// groupEntries reads group database lines, name:password:gid:members, as name to number. -func groupEntries(raw string) map[string]int { - out := map[string]int{} - for _, line := range strings.Split(raw, "\n") { - fields := strings.Split(strings.TrimSpace(line), ":") - if len(fields) < 3 || strings.HasPrefix(fields[0], "#") { - continue - } - if gid, err := strconv.Atoi(fields[2]); err == nil { - out[fields[0]] = gid - } - } - return out -} - -// GrantRefusal is why a path's group is never granted (novox/hq ADR 0312 §4), or nil when it may be: group 0, a -// group whose members act as root by membership alone — judged by its NUMBER against every number those groups -// have here, and by its name — or a group this machine names nothing for, which cannot be judged at all. -func GrantRefusal(g PathGroup) error { - const never = "a shared path is never a way to root (novox/hq ADR 0312 §4); nothing was granted, and nothing " + - "was changed on it" - if g.GID == 0 { - return fmt.Errorf("it is owned by group 0 (%s), and a read-write access never grants the root group: %s", - orRoot(g.Name), never) - } - if root, is := RootGroupIDs()[g.GID]; is { - return fmt.Errorf("it is owned by gid %d, which is %s here, whose members act as root, and a read-write "+ - "access never grants such a group: %s", g.GID, root, never) - } - if slices.Contains(accounts.RootGroups, g.Name) { - return fmt.Errorf("it is owned by the group %s, whose members act as root, and a read-write access never "+ - "grants such a group: %s", g.Words(), never) - } - if g.Name == "" { - return fmt.Errorf("it is owned by gid %d, which this machine's group database names nothing, so whether "+ - "its members act as root cannot be judged, and an unjudged group is never granted: %s", g.GID, never) - } - return nil -} - -// CannotWriteWords is the reason an access whose group may not write its path is stated unhealthy with (novox/hq -// ADR 0312 §2). Never the path: the controller says a reason on the operator's channel, which withholds a path -// (ADR 0234 §6), and keeps the target as evidence. -func CannotWriteWords(g PathGroup) string { - return fmt.Sprintf("its group cannot write it: %s owns it without group write, and the mesh changes nothing on "+ - "an accessed path (novox/hq ADR 0312 §2) — whether its group may write is the operator's to decide", - g.Words()) -} - -func orRoot(name string) string { - if name == "" { - return "root" - } - return name -} - -// grant is what one read-write access grants, read once per apply. -type grant struct { - path string - group PathGroup - // refused is why nothing is granted: the path could not be read, or its group is never granted. - refused error -} - -// grants says the group is given: it may be granted, and it may write the path. A group that cannot write is -// granted nothing (novox/hq ADR 0312 §2) — the access is stated unhealthy for the operator to decide. -func (g grant) grants() bool { return g.refused == nil && g.group.Writable } - -// grantsOf reads the group of every read-write access that names a grant, by access id. One that is refused -// grants nothing — the access itself fails, saying why — and one whose path the engine refuses to place -// anything at (issue 339) is not read at all. -func grantsOf(resources []declaration.Resource, refused map[string]error) map[string]grant { - out := map[string]grant{} - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite { - continue - } - if why := refused[filepath.Clean(a.Path)]; why != nil { - out[a.ID] = grant{path: a.Path, refused: why} - continue - } - g, err := GroupOfPath(a.Path) - if err == nil { - err = GrantRefusal(g) - } - out[a.ID] = grant{path: a.Path, group: g, refused: err} - } - return out -} - -// containerGroups is, for each container, the number of every group a read-write access grants it, sorted and -// once each: what it is created with as supplementary groups. -func containerGroups(resources []declaration.Resource, grants map[string]grant) map[string][]int { - out := map[string][]int{} - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - for _, id := range a.Grant.Containers { - if !slices.Contains(out[id], g.group.GID) { - out[id] = append(out[id], g.group.GID) - } - } - } - for id := range out { - sort.Ints(out[id]) - } - return out -} - -// ContainerGroupsOf is containerGroups for a whole declaration, read now: what the scheduler arms a scheduled -// step with, so a fire runs with the groups an apply would have created it with. A path issue 339's guard refuses -// is not read, as the apply reads none. -func ContainerGroupsOf(d *declaration.Declaration) map[string][]int { - if d == nil { - return nil - } - return containerGroups(d.Resources, grantsOf(d.Resources, refusedPlaces(d.Resources))) -} - -// withGrants adds to what the declaration asks of each account the groups its read-write accesses grant (novox/hq -// ADR 0312 §5): one user resource or access going leaves an account in a group another still asks for. -func (w Wanted) withGrants(resources []declaration.Resource, grants map[string]grant) Wanted { - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Grant.Account == "" { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - if w[a.Grant.Account] == nil { - w[a.Grant.Account] = map[string][]string{} - } - w[a.Grant.Account][g.group.Name] = append(w[a.Grant.Account][g.group.Name], a.ID) - } - return w -} - -// GrantedAccounts is every account a read-write access puts in its path's group, as the account judge reads -// them (novox/hq ADR 0312 §6): judged like a user resource's account groups, so a session of the account's own -// manager that began before is said *relogin needed*, and only there. Read now; a path whose group is refused -// or cannot be read asks nothing of the judge — the access says that itself. -func GrantedAccounts(d *declaration.Declaration, held map[string]bool) []accounts.Account { - if d == nil { - return nil - } - grants := grantsOf(d.Resources, refusedPlaces(d.Resources)) - var out []accounts.Account - for _, r := range d.Resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Grant.Account == "" || held[a.ID] { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - at := strings.LastIndex(a.ID, ".") - if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { - continue - } - out = append(out, accounts.Account{Module: a.ID[:at], ID: a.ID, Name: a.Grant.Account, - Groups: []string{g.group.Name}}) - } - sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) - return out -} - -// grantAccess is applyAccess's grant (novox/hq ADR 0312): the account put in the path's group, what an earlier -// apply of this access put an account in and nobody asks for any more taken back, and the outcome said. The -// path itself is only read. -func grantAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, - in inputs, out *Outcome) error { - var g grant - granting := r.Grant != nil && r.Mode == declaration.AccessReadWrite - if granting { - var read bool - g, read = in.grants[r.ID] - if !read { - // An apply that read nothing ahead — a caller with no inputs — reads it here. - g = grantsOf([]declaration.Resource{r}, in.refused)[r.ID] - } - if g.refused != nil { - return fmt.Errorf("%s grants nothing: %w", r.Path, g.refused) - } - } - // A group that cannot write is granted nothing (§2): what an earlier apply granted is taken back below. - account := "" - if granting && g.group.Writable { - account = r.Grant.Account - } - if account != "" && g.group.Name == "" { - return fmt.Errorf("%s is owned by %s, which this machine's group database names nothing, and an account "+ - "is put in a group by name: %s was not (novox/hq ADR 0312 §1)", r.Path, g.group.Words(), account) - } - - var said []string - moved := false - // What this access put an account in before. An access whose account changed gives the old one back - // everything it put it in; the new one starts with nothing recorded. - recorded := previous.Groups - if previous.User != "" && previous.User != account { - gave, words := takeBack(ctx, sys, previous.User, r.ID, previous.Groups, nil, run, in.groups) - moved = moved || gave - said = append(said, words...) - recorded = nil - } - var kept []string - if account != "" { - groups, err := system.GroupsOf(ctx, system.Runner(run), account) - if err != nil { - return err - } - inGroup := setOf(groups)[g.group.Name] - if !inGroup { - if err := sys.AddUserToGroup(ctx, system.Runner(run), account, g.group.Name); err != nil { - return err - } - moved = true - said = append(said, fmt.Sprintf("put %s in %s, the group that owns it; the tool runner holds it once "+ - "restarted, a session that began before only after a new login", account, g.group.Name)) - } - // Recorded only when the mesh put it there, now or before, or a record that went handed it over: a group - // held before is never taken back. - if !inGroup || slices.Contains(recorded, g.group.Name) || in.handed.has(account, g.group.Name) { - kept = append(kept, g.group.Name) - } - var others []string - for _, x := range recorded { - if x != g.group.Name { - others = append(others, x) - } - } - gave, words := takeBack(ctx, sys, account, r.ID, others, &kept, run, in.groups) - moved = moved || gave - said = append(said, words...) - } - if len(kept) > 0 { - out.groups, out.user = kept, account - } - - if granting { - var to []string - if len(r.Grant.Containers) > 0 { - to = append(to, "its containers "+strings.Join(r.Grant.Containers, ", ")) - } - if account != "" { - to = append(to, account) - } - switch { - case !g.group.Writable: - out.Detail = joinDetail(out.Detail, fmt.Sprintf("its group %s cannot write it, so nothing is granted: "+ - "stated unhealthy, and nothing was changed on it — the operator decides", g.group.Words())) - case len(to) > 0: - out.Detail = joinDetail(out.Detail, fmt.Sprintf("grants %s to %s", g.group.Words(), strings.Join(to, " and "))) - } - } - if len(said) > 0 { - out.Detail = joinDetail(out.Detail, strings.Join(said, "; ")) - } - if moved { - // Updated exactly when the account's groups moved: what names this access under restart-on — the - // tool runner — is restarted to hold them (novox/hq ADR 0312 §6). - out.Action = "updated" - } - return nil -} - -// takeBack takes account out of every group of groups the mesh put it in for the resource id, unless another -// declared resource still asks for it — then it stays, and stays recorded on keep when keep is given, so it is -// taken back once nobody asks. A group it is no longer in is nothing to give back. Never fatal: a failure is -// said, and the group stays recorded where it can be. Whether any group was taken back, and what was said. -func takeBack(ctx context.Context, sys system.System, account, id string, groups []string, keep *[]string, - run Runner, wanted Wanted) (bool, []string) { - if len(groups) == 0 { - return false, nil - } - in, err := system.GroupsOf(ctx, system.Runner(run), account) - if err != nil { - if keep != nil { - *keep = append(*keep, groups...) - } - return false, []string{fmt.Sprintf("the groups the mesh put %s in (%s) were not given back: %v", account, - strings.Join(groups, ", "), err)} - } - already := setOf(in) - gave := false - var said []string - for _, g := range groups { - if !already[g] { - continue - } - if others := wanted.othersAsk(account, g, id); len(others) > 0 { - said = append(said, fmt.Sprintf("left %s in %s, which %s still asks for", account, g, strings.Join(others, ", "))) - if keep != nil { - *keep = append(*keep, g) - } - continue - } - gone, why := leaveGroup(ctx, sys, account, g, run) - if !gone && keep != nil { - *keep = append(*keep, g) - } - gave = gave || gone - said = append(said, account+" "+why) - } - return gave, said -} - -// removeAccess is an access no longer declared: the path left exactly as it is (novox/hq ADR 0051), and every -// group it put an account in taken back unless another declared resource still asks for it (ADR 0312 §5) — then -// handed to that resource, which records it as the mesh's and takes it back when it stops asking. -func removeAccess(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted, - handed Handed) (string, string) { - const left = "an operator-owned path is never the host's to remove" - if a.User == "" || len(a.Groups) == 0 { - return "forgotten", left - } - var stays []string - gave, said := takeBack(ctx, sys, a.User, a.ID, a.Groups, &stays, run, wanted) - for _, g := range stays { - if len(wanted.othersAsk(a.User, g, a.ID)) > 0 { - handed.hand(a.User, g) - } - } - action := "forgotten" - if gave { - action = "restored" - } - return action, joinDetail(left, strings.Join(said, "; ")) -} - -// groupAddArgs is the runtime's arguments that give a container's process each granted group as a supplementary -// group, by number: the container's own group database is not the machine's, and a name could mean another -// group there, or none. -func groupAddArgs(groups []int) []string { - var args []string - for _, gid := range groups { - args = append(args, "--group-add", strconv.Itoa(gid)) - } - return args -} - -func gidWords(groups []int) string { - words := make([]string, 0, len(groups)) - for _, gid := range groups { - words = append(words, strconv.Itoa(gid)) - } - return strings.Join(words, ", ") -} - -// Handed is every group a record that went in this apply put an account in, and another declared resource still -// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group -// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it -// there went first (novox/hq ADR 0252 §3, ADR 0312 §5). -type Handed map[string]map[string]bool - -func (h Handed) hand(account, group string) { - if h == nil { - return - } - if h[account] == nil { - h[account] = map[string]bool{} - } - h[account][group] = true -} - -func (h Handed) has(account, group string) bool { return h[account][group] } diff --git a/internal/apply/grants_test.go b/internal/apply/grants_test.go deleted file mode 100644 index e540a4d..0000000 --- a/internal/apply/grants_test.go +++ /dev/null @@ -1,559 +0,0 @@ -package apply - -import ( - "context" - "errors" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/accounts" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" -) - -// Defends novox/hq ADR 0312 and issue 537: a read-write access grants its path's own group to the containers and -// the account the controller names, and the path is never changed; a read access grants nothing; the root group, -// and a group whose members act as root, are refused by name; what was granted is taken back as ADR 0252 takes -// back; a container is recreated with the group, and a unit of the machine's own manager running as the account -// is restarted once the account holds it. - -// grantMachine is the group database of groupDB, a container runtime that keeps each container's spec label and -// the arguments it was run with, and a service manager that notes, at each restart, the groups the account holds -// in the database then — which is what the machine's own manager gives the unit's new process. -type grantMachine struct { - *groupDB - containers map[string]string - runs map[string][]string - restarts []string -} - -func newGrantMachine(account string, in []string, exists ...string) *grantMachine { - g := newGroupDB(in, exists...) - g.account = account - return &grantMachine{groupDB: g, containers: map[string]string{}, runs: map[string][]string{}} -} - -func (m *grantMachine) run(ctx context.Context, name string, args ...string) (string, error) { - a := oldShape(name, args) - switch name { - case "docker": - m.asked = append(m.asked, name+" "+strings.Join(a, " ")) - switch a[0] { - case "info": - return "27.0\n", nil - case "container": - spec, there := m.containers[a[len(a)-1]] - if !there { - return "", errors.New("no such container") - } - return "true\t" + spec, nil - case "rm": - delete(m.containers, a[len(a)-1]) - case "run": - var named, spec string - for i := 0; i+1 < len(a); i++ { - switch { - case a[i] == "--name": - named = a[i+1] - case a[i] == "--label" && strings.HasPrefix(a[i+1], specLabel+"="): - spec = strings.TrimPrefix(a[i+1], specLabel+"=") - } - } - m.containers[named] = spec - m.runs[named] = a - return "deadbeef\n", nil - } - return "", nil - case "systemctl": - m.asked = append(m.asked, name+" "+strings.Join(a, " ")) - if a[0] == "restart" { - m.restarts = append(m.restarts, a[1]+": "+m.groups()) - } - return "", nil - } - return m.groupDB.run(ctx, name, args...) -} - -// groupOfPaths stands in for the machine's group database and the paths' groups for the length of a test. -func groupOfPaths(t *testing.T, groups map[string]PathGroup) { - t.Helper() - was := GroupOfPath - GroupOfPath = func(path string) (PathGroup, error) { - if _, err := os.Stat(path); err != nil { - return PathGroup{}, err - } - g, ok := groups[path] - if !ok { - return PathGroup{}, errors.New("no group in this test for " + path) - } - return g, nil - } - t.Cleanup(func() { GroupOfPath = was }) - rootGroupsAre(t, map[int]string{0: "root", 10: "wheel", 968: "docker"}) -} - -// rootGroupsAre stands in for the numbers this machine gives the groups whose members act as root. -func rootGroupsAre(t *testing.T, ids map[int]string) { - t.Helper() - was := RootGroupIDs - RootGroupIDs = func() map[int]string { return ids } - t.Cleanup(func() { RootGroupIDs = was }) -} - -func accessGranting(id, path, account string, containers ...string) string { - grant := `"grant":{"account":"` + account + `"` - if len(containers) > 0 { - grant += `,"containers":["` + strings.Join(containers, `","`) + `"]` - } - return `{"id":"` + id + `","type":"access","path":"` + path + `","mode":"read-write",` + grant + `}}` -} - -func mounting(id, path string) string { - return `{"id":"` + id + `","type":"container","name":"` + strings.ReplaceAll(id, ".", "-") + `","image":"` + pinned + - `","volumes":["` + path + `:/library"]}` -} - -func applyGrants(t *testing.T, m *grantMachine, known store.State, resources ...string) (Report, store.State, error) { - t.Helper() - return Apply(context.Background(), archHost(t), parse(t, `{"declaration":1,"resources":[`+ - strings.Join(resources, ",")+`]}`), known, store.OriginDeclared, m.run, nil, nil) -} - -func statOf(t *testing.T, path string) string { - t.Helper() - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - uid, gid, _ := OwnerOf(info) - return fmt.Sprintf("%s %d:%d %s", info.Mode(), uid, gid, info.ModTime()) -} - -func TestAReadWriteAccessGrantsItsPathsGroupToTheContainerAndTheAccountAndLeavesThePath(t *testing.T) { - library := t.TempDir() - if err := os.WriteFile(filepath.Join(library, "album"), []byte("x"), 0o640); err != nil { - t.Fatal(err) - } - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - before, beforeInside := statOf(t, library), statOf(t, filepath.Join(library, "album")) - m := newGrantMachine("operator", []string{"operator"}, "media") - - report, state, err := applyGrants(t, m, store.State{}, - accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(" "+m.groups()+" ", " media ") { - t.Fatalf("the account was not put in the path's group: %q", m.groups()) - } - if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") { - t.Fatalf("the container was not created with the path's group: %s", run) - } - o := outcomeOf(report, "importer.music") - if o.Action != "updated" || !strings.Contains(o.Detail, "grants media (gid 1001)") || - !strings.Contains(o.Detail, "put operator in media") { - t.Errorf("the access did not say what it granted: %+v", o) - } - if rec, _ := state.Find("importer.music"); rec.User != "operator" || strings.Join(rec.Groups, " ") != "media" { - t.Errorf("the access did not record the group it put the account in: %+v", rec) - } - // The path, and what is under it, exactly as they were (ADR 0312 §2). - if after := statOf(t, library); after != before { - t.Errorf("the accessed path changed: %s, was %s", after, before) - } - if after := statOf(t, filepath.Join(library, "album")); after != beforeInside { - t.Errorf("a file under the accessed path changed: %s, was %s", after, beforeInside) - } - for _, asked := range m.asked { - for _, verb := range []string{"chown", "chmod", "chgrp", "setfacl"} { - if strings.HasPrefix(asked, verb) || strings.Contains(asked, " "+verb+" ") { - t.Errorf("the engine asked the machine to change something: %q", asked) - } - } - } - - // Applied again: nothing moves, and nothing is recreated. - m.runs = map[string][]string{} - report, _, err = applyGrants(t, m, state, - accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if o := outcomeOf(report, "importer.music"); o.Action != "unchanged" { - t.Errorf("a second apply moved the access: %+v", o) - } - if len(m.runs) != 0 { - t.Errorf("a second apply recreated the container: %v", m.runs) - } -} - -func TestAReadAccessGrantsNothing(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - - // A grant on a read access is refused whole: the controller asked for something the decision forbids. - _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"player.music","type":"access","path":"` + - library + `","mode":"read","grant":{"account":"operator"}}]}`)) - if err == nil || !strings.Contains(err.Error(), "a read access grants nothing") { - t.Fatalf("a grant on a read access was not refused by name: %v", err) - } - - m := newGrantMachine("operator", []string{"operator"}, "media") - report, state, err := applyGrants(t, m, store.State{}, - `{"id":"player.music","type":"access","path":"`+library+`","mode":"read"}`, mounting("player.app", library)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("a read access put the account in a group: %q", m.groups()) - } - if run := strings.Join(m.runs["player-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("a read access gave its container a group: %s", run) - } - if o := outcomeOf(report, "player.music"); o.Action != "unchanged" { - t.Errorf("a read access moved something: %+v", o) - } - if rec, _ := state.Find("player.music"); len(rec.Groups) != 0 || rec.User != "" { - t.Errorf("a read access recorded a grant: %+v", rec) - } -} - -func TestAnAccessWhoseGroupCannotWriteGrantsNothingAndSaysTheOperatorDecides(t *testing.T) { - spool := t.TempDir() - groupOfPaths(t, map[string]PathGroup{spool: {GID: 1002, Name: "downloads", Writable: false}}) - before := statOf(t, spool) - m := newGrantMachine("operator", nil, "downloads") - - report, _, err := applyGrants(t, m, store.State{}, - accessGranting("nzbget.downloads", spool, "operator", "nzbget.app"), mounting("nzbget.app", spool)) - if err != nil { - t.Fatal(err) - } - o := outcomeOf(report, "nzbget.downloads") - if !strings.Contains(o.Detail, "cannot write it, so nothing is granted") || !strings.Contains(o.Detail, "the operator decides") || - !strings.Contains(o.Detail, "nothing was changed") { - t.Errorf("the access did not say its group cannot write and that the operator decides: %+v", o) - } - // Nothing is given (ADR 0312 §2): no account group, no container group. - if strings.Contains(m.groups(), "downloads") { - t.Errorf("a group that cannot write was granted to the account: %q", m.groups()) - } - if run := strings.Join(m.runs["nzbget-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("a group that cannot write was granted to the container: %s", run) - } - if after := statOf(t, spool); after != before { - t.Errorf("the engine changed a path whose group cannot write: %s, was %s", after, before) - } - // The words the look states it unhealthy with: the remedy is the operator's, and the path is never in them. - words := CannotWriteWords(PathGroup{GID: 1002, Name: "downloads"}) - if !strings.Contains(words, "the operator's to decide") || strings.Contains(words, spool) { - t.Errorf("the unhealthy reason is %q", words) - } -} - -func TestTheRootGroupIsNeverGranted(t *testing.T) { - for _, g := range []PathGroup{{GID: 0, Name: "root", Writable: true}, {GID: 968, Name: "docker", Writable: true}, - {GID: 10, Name: "wheel", Writable: true}, {GID: 42, Name: "shadow", Writable: true}, - // A number the root groups have here under another name — judged by the number, never the name. - {GID: 968, Name: "containers", Writable: true}, - // A number the machine names nothing for: unjudged, so never granted, to a container either. - {GID: 4242, Name: "", Writable: true}} { - want := g.Name - if want == "" || want == "containers" { - want = fmt.Sprintf("gid %d", g.GID) - } - t.Run(fmt.Sprintf("%d %s", g.GID, g.Name), func(t *testing.T) { - shared := t.TempDir() - groupOfPaths(t, map[string]PathGroup{shared: g}) - m := newGrantMachine("operator", nil, g.Name) - - report, _, err := applyGrants(t, m, store.State{}, - accessGranting("importer.shared", shared, "operator", "importer.app"), mounting("importer.app", shared)) - if err == nil { - t.Fatal("a read-write access to a path of a root-granting group applied") - } - var failed *Error - if !errors.As(err, &failed) || failed.Resource != "importer.shared" || - !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never") { - t.Errorf("the refusal does not name the access and the group: %v", err) - } - if g.Name != "" && strings.Contains(m.groups(), g.Name) { - t.Errorf("the account was put in %s", g.Name) - } - if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("the container was given a root-granting group: %s", run) - } - _ = report - }) - } -} - -func TestAGrantedGroupIsTakenBackUnlessStillAskedForOrHeldBefore(t *testing.T) { - music, movies := t.TempDir(), t.TempDir() - groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}, - movies: {GID: 1001, Name: "media", Writable: true}}) - - t.Run("no longer asked", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - report, state, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())) - if err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("the group the mesh granted was not taken back: %q", m.groups()) - } - if o := outcomeOf(report, "importer.music"); o.Action != "restored" || !strings.Contains(o.Detail, "taken out of media") { - t.Errorf("the removal did not say it took the group back: %+v", o) - } - if _, still := state.Find("importer.music"); still { - t.Error("the access is still recorded") - } - }) - - t.Run("still asked by another access", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), - accessGranting("sonarr.movies", movies, "operator")) - if err != nil { - t.Fatal(err) - } - report, _, err := applyGrants(t, m, state, accessGranting("sonarr.movies", movies, "operator")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group another access still grants was taken back: %q", m.groups()) - } - if o := outcomeOf(report, "importer.music"); !strings.Contains(o.Detail, "still asks for") { - t.Errorf("the removal did not say why the group stayed: %+v", o) - } - }) - - t.Run("still asked by a user resource", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - if _, _, err := applyGrants(t, m, state, - `{"id":"player.account","type":"user","name":"operator","groups":["media"]}`); err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group a user resource asks for was taken back: %q", m.groups()) - } - }) - - t.Run("held before", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator", "media"}) - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - if rec, _ := state.Find("importer.music"); len(rec.Groups) != 0 { - t.Fatalf("a group the account held before was recorded as the mesh's: %+v", rec) - } - if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group the account held before the mesh was taken back: %q", m.groups()) - } - }) -} - -func TestAContainerIsRecreatedWithTheGroupItIsGranted(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - m := newGrantMachine("operator", nil, "media") - - // Running before its access granted anything: a controller older than the grant. - _, state, err := applyGrants(t, m, store.State{}, - `{"id":"importer.music","type":"access","path":"`+library+`","mode":"read-write"}`, mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") { - t.Fatalf("a container granted nothing was given a group: %s", run) - } - report, _, err := applyGrants(t, m, state, - accessGranting("importer.music", library, "", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - o := outcomeOf(report, "importer.app") - if o.Action != "updated" || !strings.Contains(o.Detail, "recreated to hold the groups") { - t.Errorf("the container was not recreated for its group: %+v", o) - } - if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") { - t.Errorf("the recreated container does not hold the group: %s", run) - } -} - -func TestTheToolRunnerIsRestartedOnceItsAccountHoldsTheGroup(t *testing.T) { - me, err := user.Current() - if err != nil { - t.Skip("no current account to run the tool runner as") - } - units, bundles := t.TempDir(), t.TempDir() - wasUnits, wasBundles := unitDir, daemonRoot - unitDir, daemonRoot = units, bundles - t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles }) - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - m := newGrantMachine(me.Username, nil, "media") - - body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"}) - runner := `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) + - `","digest":"` + digest + `","run":["node","main.js"],"user":"` + me.Username + - `","restart-on":["importer.music"]}` - // Running before the grant. - _, state, err := applyGrants(t, m, store.State{}, runner) - if err != nil { - t.Fatal(err) - } - m.restarts = nil - report, state, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner) - if err != nil { - t.Fatal(err) - } - if len(m.restarts) != 1 || !strings.HasPrefix(m.restarts[0], "node-tools.service: ") || - !strings.Contains(m.restarts[0], "media") { - t.Fatalf("the tool runner was not restarted once its account held the group: %v", m.restarts) - } - if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "importer.music") { - t.Errorf("the restart does not say what it was for: %+v", o) - } - // And not again while nothing moves. - m.restarts = nil - if _, _, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner); err != nil { - t.Fatal(err) - } - if len(m.restarts) != 0 { - t.Errorf("the tool runner was restarted with nothing moved: %v", m.restarts) - } -} - -// sessionReader is an account whose own service manager runs, holding the groups it began with. -type sessionReader struct{ database, session []string } - -func (s sessionReader) InDatabase(context.Context, string) ([]string, error) { return s.database, nil } -func (s sessionReader) Session(_ context.Context, _ string, groups []string) (accounts.Session, error) { - has := map[string]bool{} - for _, g := range s.session { - has[g] = true - } - return accounts.Session{Running: true, Has: has}, nil -} -func (s sessionReader) Escalation(context.Context, string, []string) ([]string, error) { - return nil, nil -} - -func TestAnAccountsOwnSessionIsSaidReloginNeededForAGrantedGroup(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - d := parse(t, `{"declaration":1,"resources":[`+accessGranting("importer.music", library, "operator")+`]}`) - - as := GrantedAccounts(d, nil) - if len(as) != 1 || as[0].Module != "importer" || as[0].Name != "operator" || strings.Join(as[0].Groups, " ") != "media" { - t.Fatalf("the account the access grants is not judged: %+v", as) - } - j := accounts.New(sessionReader{database: []string{"operator", "media"}, session: []string{"operator"}}) - j.Set(as) - st, _ := j.Look(context.Background()) - if len(st.Accounts) != 1 || !strings.HasPrefix(st.Accounts[0].Reason, accounts.ReasonRelogin) { - t.Errorf("a session of the account's own manager that began before the grant is not said relogin needed: %+v", - st.Accounts) - } - // A read access, or one refused, asks nothing of the judge. - groupOfPaths(t, map[string]PathGroup{library: {GID: 0, Name: "root", Writable: true}}) - if as := GrantedAccounts(d, nil); len(as) != 0 { - t.Errorf("a refused grant is judged: %+v", as) - } -} - -// An access going while a user resource still asks for its group hands the group over: the user resource records it -// as the mesh's, and takes it back when it stops asking too (novox/hq ADR 0312 §5, ADR 0252 §3). -func TestAGroupStillAskedForIsHandedOverAndTakenBackWhenTheLastAskerGoes(t *testing.T) { - music := t.TempDir() - groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}}) - player := `{"id":"player.account","type":"user","name":"operator","groups":["media"]}` - m := newGrantMachine("operator", []string{"operator"}, "media") - - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), player) - if err != nil { - t.Fatal(err) - } - if rec, _ := state.Find("player.account"); len(rec.Groups) != 0 { - t.Fatalf("the user resource recorded a group the access put the account in: %+v", rec) - } - _, state, err = applyGrants(t, m, state, player) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Fatalf("a group the user resource still asks for was taken back: %q", m.groups()) - } - if rec, _ := state.Find("player.account"); strings.Join(rec.Groups, " ") != "media" { - t.Fatalf("the group was not handed to the user resource that still asks for it: %+v", rec) - } - if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("the group stayed after the last resource asking for it went: %q", m.groups()) - } -} - -// A grant's account is an argument to id, usermod and gpasswd, and a name that starts with a dash would be read as -// an option by each. -func TestAGrantsAccountMayNotReadAsAnOption(t *testing.T) { - for _, name := range []string{"-ogroups", "--help", "a b", "a:b"} { - _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` + - `"mode":"read-write","grant":{"account":"` + name + `"}}]}`)) - if err == nil || !strings.Contains(err.Error(), "not starting with a dash") { - t.Errorf("the account %q was accepted: %v", name, err) - } - } - if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` + - `"mode":"read-write","grant":{"account":"jochen"}}]}`)); err != nil { - t.Errorf("an ordinary account was refused: %v", err) - } -} - -// The scheduler reads no path issue 339's guard refuses, as the apply reads none. -func TestAScheduledStepIsArmedWithoutReadingARefusedPath(t *testing.T) { - var read []string - was := GroupOfPath - GroupOfPath = func(path string) (PathGroup, error) { - read = append(read, path) - return PathGroup{GID: 1001, Name: "media", Writable: true}, nil - } - t.Cleanup(func() { GroupOfPath = was }) - rootGroupsAre(t, map[int]string{0: "root"}) - d := parse(t, `{"declaration":1,"resources":[`+accessGranting("m.etc", "/etc", "", "m.step")+`,`+ - `{"id":"m.step","type":"container","name":"step","image":"`+pinned+`","schedule":"0 3 * * *","volumes":["/etc:/x"]}]}`) - if got := ContainerGroupsOf(d); len(got["m.step"]) != 0 { - t.Errorf("a step was armed with the group of a refused path: %v", got) - } - if len(read) != 0 { - t.Errorf("the scheduler read a path the guard refuses: %v", read) - } -} - -// The group database is read as name to number, so a number carrying a root group's name under any other name -// is still that number. -func TestGroupEntriesAreReadByNumber(t *testing.T) { - got := groupEntries("root:x:0:\n# a comment\ndocker:x:968:operator\ncontainers:x:968:\nbroken\n") - if got["root"] != 0 || got["docker"] != 968 || got["containers"] != 968 || len(got) != 3 { - t.Errorf("read %v", got) - } -} diff --git a/internal/apply/identities.go b/internal/apply/identities.go new file mode 100644 index 0000000..5fb229f --- /dev/null +++ b/internal/apply/identities.go @@ -0,0 +1,253 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Every module is given an identity of its own (novox/hq ADR 0325 §3, §11, issue 567). +// +// The controller allocates each module a user and a group number once, from a range of its own, the same on every +// machine, and declares them as `mesh-`. The engine makes the group and the account with exactly those +// numbers — a system account: no login shell, no home, no password — before anything of the declaration is +// applied, so a directory the module owns can be given to it and a container told to run as it. +// +// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is +// refused for the whole declaration before anything is made or started, naming why — for an account and for an +// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is +// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is. +// +// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the +// engine made the account, the group or both, and an account or a group that was there before is never removed. + +// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the +// runner, which is what a test stands in for. +type accountsTool interface { + CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int) error + CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error + RemoveAccount(ctx context.Context, run system.Runner, name string) error + RemoveGroup(ctx context.Context, run system.Runner, name string) error +} + +// entry is one line the user or group database gives for a key: its name and its numbers. +type entry struct { + name string + id, prim int +} + +// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a +// database that does not answer is an error, never read as absent (system.LookUpUser's rule). +func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) { + out, err := run(ctx, "getent", database, key) + if err != nil { + if code, ok := system.ExitCode(err); ok && code == 2 { + return entry{}, false, nil + } + return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err) + } + fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":") + if len(fields) < 3 { + return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key) + } + e := entry{name: fields[0], prim: -1} + if e.id, err = strconv.Atoi(fields[2]); err != nil { + return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key) + } + if database == "passwd" && len(fields) >= 4 { + if e.prim, err = strconv.Atoi(fields[3]); err != nil { + e.prim = -1 + } + } + return e, true, nil +} + +// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before +// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started. +func refuseRootIdentities(d *declaration.Declaration) error { + var problems []string + for _, id := range d.Identities { + if err := rootIdentity(id.UID, id.GID); err != nil { + problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err)) + } + } + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && c.RunAs != nil { + if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil { + problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v", + c.ID, c.RunAs.UID, c.RunAs.GID, err)) + } + } + } + if len(problems) == 0 { + return nil + } + sort.Strings(problems) + return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; ")) +} + +// applyIdentity makes one identity as declared, or refuses it naming whose the name or the numbers are. What it +// made is recorded on known; an identity already right is unchanged. +func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, known *store.State, + run Runner) (Outcome, error) { + out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"} + user, hasUser, err := lookUp(ctx, run, "passwd", id.Name) + if err != nil { + return out, err + } + group, hasGroup, err := lookUp(ctx, run, "group", id.Name) + if err != nil { + return out, err + } + byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID)) + if err != nil { + return out, err + } + byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.GID)) + if err != nil { + return out, err + } + const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0325 §3)" + switch { + case hasUser && (user.id != id.UID || user.prim != id.GID): + return out, fmt.Errorf("the account %s is here with uid %d and group %d, and the mesh allocated uid %d, "+ + "gid %d%s", id.Name, user.id, user.prim, id.UID, id.GID, never) + case hasGroup && group.id != id.GID: + return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d%s", id.Name, + group.id, id.GID, never) + case uidUsed && byUID.name != id.Name: + return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID, + byUID.name, id.Name, never) + case gidUsed && byGID.name != id.Name: + return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.GID, + byGID.name, id.Name, never) + } + rec := madeIdentity(known, id.Name) + rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID + var made []string + if !hasGroup { + if err := tool.CreateSystemGroup(ctx, run, id.Name, id.GID); err != nil { + return out, err + } + rec.MadeGroup = true + made = append(made, fmt.Sprintf("the group (gid %d)", id.GID)) + } + if !hasUser { + if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID); err != nil { + recordIdentity(known, rec) + return out, err + } + rec.MadeUser = true + made = append(made, fmt.Sprintf("the account (uid %d), with no login, no home and no password", id.UID)) + } + if rec.At.IsZero() { + rec.At = time.Now().UTC() + } + recordIdentity(known, rec) + if len(made) > 0 { + out.Action, out.Detail = "created", "made "+strings.Join(made, " and ") + } else { + out.Detail = "here with its numbers" + } + return out, nil +} + +// madeIdentity is what the engine recorded of name, or an empty record. +func madeIdentity(known *store.State, name string) store.MadeIdentity { + for _, m := range known.Identities { + if m.Name == name { + return m + } + } + return store.MadeIdentity{} +} + +// recordIdentity records what was made of one identity, in place of any record before. +func recordIdentity(known *store.State, rec store.MadeIdentity) { + for i, m := range known.Identities { + if m.Name == rec.Name { + known.Identities[i] = rec + return + } + } + if rec.MadeUser || rec.MadeGroup { + known.Identities = append(known.Identities, rec) + } +} + +// removeIdentities removes what the engine made of every identity no longer declared — the account, then the +// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed +// stays recorded, said, for the next apply. +func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State, + run Runner) []Outcome { + wanted := map[string]bool{} + for _, id := range declared { + wanted[id.Name] = true + } + var out []Outcome + var keep []store.MadeIdentity + for _, m := range known.Identities { + if wanted[m.Name] { + keep = append(keep, m) + continue + } + o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"} + var said []string + if m.MadeUser { + if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) { + said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+ + "database did not answer") + m.MadeUser = false + } else if has { + if err := tool.RemoveAccount(ctx, run, m.Name); err != nil { + said = append(said, "the account could not be removed: "+err.Error()) + } else { + m.MadeUser = false + said = append(said, "removed the account") + } + } else { + m.MadeUser = false + } + } + if m.MadeGroup && !m.MadeUser { + if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) { + said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+ + "database did not answer") + m.MadeGroup = false + } else if has { + if err := tool.RemoveGroup(ctx, run, m.Name); err != nil { + said = append(said, "the group could not be removed: "+err.Error()) + } else { + m.MadeGroup = false + said = append(said, "removed the group") + } + } else { + m.MadeGroup = false + } + } + if m.MadeUser || m.MadeGroup { + o.Action = "waiting" + keep = append(keep, m) + } + o.Detail = "no longer declared: " + strings.Join(said, "; ") + out = append(out, o) + } + known.Identities = keep + return out +} + +// toolFor is the accounts tool of a system, or why it has none. +func toolFor(sys system.System) (accountsTool, error) { + if t, ok := sys.(accountsTool); ok { + return t, nil + } + return nil, errors.New("this system makes no module identity: it has no way to make a system account") +} diff --git a/internal/apply/identities_test.go b/internal/apply/identities_test.go new file mode 100644 index 0000000..023e088 --- /dev/null +++ b/internal/apply/identities_test.go @@ -0,0 +1,287 @@ +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with +// the numbers the controller allocated, a container told to run as it, and never a root number. + +// accountsMachine is a user and group database the runner answers from, and every command that changed it. +type accountsMachine struct { + users, groups map[string][2]int // name → id, primary gid (a group's second is unused) + ran []string +} + +func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + switch name { + case "getent": + db, key := args[0], args[1] + table := m.users + if db == "group" { + table = m.groups + } + for n, ids := range table { + if n == key || strconv.Itoa(ids[0]) == key { + if db == "group" { + return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil + } + return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::/:/usr/bin/nologin\n", nil + } + } + return "", errors.New("getent exited 2") + case "groupadd": + m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0} + case "useradd": + m.users[args[len(args)-1]] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])} + case "userdel": + delete(m.users, args[0]) + case "groupdel": + delete(m.groups, args[0]) + default: + return "", nil + } + m.ran = append(m.ran, line) + return "", nil +} + +func mustAtoi(s string) int { + n, _ := strconv.Atoi(s) + return n +} + +func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` + + resources + `]}`)) + if err != nil { + t.Fatal(err) + } + return d +} + +// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was +// made — when no longer declared. +func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) { + m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}} + dir := t.TempDir() + d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`) + report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err != nil { + t.Fatal(err) + } + want := []string{ + "groupadd --system --gid 61001 mesh-radarr", + "useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir / --shell /usr/bin/nologin mesh-radarr", + } + if strings.Join(m.ran, "\n") != strings.Join(want, "\n") { + t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n")) + } + if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" { + t.Errorf("the identity's outcome: %+v", o) + } + if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup { + t.Fatalf("what was made is not recorded: %+v", known.Identities) + } + + // Applied again: nothing is made twice. + m.ran = nil + if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil { + t.Fatal(err) + } + if len(m.ran) != 0 { + t.Fatalf("an identity already here was made again: %v", m.ran) + } + + // Undeclared, with a group the machine had before for another identity: only what was made goes. + m.groups["mesh-sonarr"] = [2]int{61002, 0} + other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`, + `{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`) + if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil { + t.Fatal(err) + } + if _, has := m.users["mesh-radarr"]; has { + t.Error("an identity no longer declared kept its account") + } + if _, has := m.groups["mesh-radarr"]; has { + t.Error("an identity no longer declared kept its group") + } + if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup { + t.Error("a group that was there before is recorded as the mesh's") + } +} + +// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the +// module is not attempted; the rest of the machine is. +func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) { + dir := t.TempDir() + for _, tc := range []struct { + users, groups map[string][2]int + says string + }{ + {users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"}, + {users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"}, + {users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"}, + } { + m := &accountsMachine{users: tc.users, groups: tc.groups} + d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+ + `{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err == nil || !strings.Contains(err.Error(), tc.says) { + t.Errorf("not refused naming %s: %v", tc.says, err) + } + if len(m.ran) != 0 { + t.Errorf("something was made for an identity that is another's: %v", m.ran) + } + if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" { + t.Errorf("the module of a refused identity was attempted: %+v", o) + } + if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil { + t.Errorf("the rest of the machine was not applied: %v", err) + } + os.RemoveAll(filepath.Join(dir, "plex")) + } +} + +// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is +// made or started. A number 0 is refused on arrival, by the parser. +func TestNoModuleIdentityIsRoot(t *testing.T) { + was := RootGroupIDs + RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} } + defer func() { RootGroupIDs = was }() + dir := t.TempDir() + for _, d := range []*declaration.Declaration{ + withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`), + withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+ + `{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+ + `"identity":{"mode":"run","uid":61001,"gid":10}}`), + } { + m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}} + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") { + t.Errorf("a root group was not refused naming why: %v", err) + } + if len(m.ran) != 0 { + t.Errorf("something was made or started: %v", m.ran) + } + if _, err := os.Stat(filepath.Join(dir, "config")); err == nil { + t.Error("a resource of a declaration naming a root group was applied") + } + } + for _, raw := range []string{ + `{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`, + `{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` + + `"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`, + } { + if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") { + t.Errorf("a number 0 was not refused on arrival: %v", err) + } + } +} + +// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user +// option — and a changed identity is a container recreated. +func TestAContainerIsToldItsModulesIdentity(t *testing.T) { + env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{ + Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}} + run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{ + Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}} + if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") { + t.Errorf("an image told by environment was not: %s", got) + } + if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") { + t.Errorf("an image told by the runtime's user option was not: %s", got) + } + plain := *env + plain.RunAs = nil + moved := *env + moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", + UID: 61009, GID: 61009} + if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) || + containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) { + t.Error("an identity told, or renumbered, does not recreate the container") + } +} + +// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of +// the access, and nothing is granted again. +func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) { + dir := t.TempDir() + groups := "nox media" + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + switch name { + case "id": + return groups + "\n", nil + case "gpasswd": + groups = "nox" + } + return "", nil + } + known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir, + Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}} + d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`) + report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") { + t.Fatalf("the grant's group was not taken back: %v", ran) + } + if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") { + t.Errorf("the take-back is not said: %+v", o) + } + if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 { + t.Errorf("the record still holds the grant's groups: %+v", rec) + } +} + +// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again. +func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) { + dir := t.TempDir() + refuse := true + groups := "nox media" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch name { + case "id": + return groups + "\n", nil + case "gpasswd": + if refuse { + return "", errors.New("gpasswd exited 1: locked") + } + groups = "nox" + } + return "", nil + } + known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir, + Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}} + d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`) + _, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 { + t.Fatalf("a group that could not be left was forgotten: %+v", rec) + } + refuse = false + if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" { + t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups) + } +} diff --git a/internal/apply/keep.go b/internal/apply/keep.go new file mode 100644 index 0000000..27d5afc --- /dev/null +++ b/internal/apply/keep.go @@ -0,0 +1,953 @@ +package apply + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// A shared folder is kept usable through access entries (novox/hq ADR 0325, issue 567). +// +// Several modules use one folder of the operator's — a library one imports into, one renames in, one plays from — +// each as its own identity. The engine keeps, for every placed folder, one ACL entry per identity that uses it, +// on the folder and as its default for everything made under it, and changes nothing else of the path (§5). +// +// **When it may keep a folder.** Only one that is the mesh's to keep: taken on the operator's warrant (§8), or +// holding no data yet — empty when first seen, or made by the engine (§9). A folder that holds data and was never +// taken is stated waiting, with a wait of kind access, until the operator lets the mesh keep it (§7); one whose disk +// is not there waits for the disk (§9); root's data on its placed filesystem is never kept, and is unhealthy (§12). +// +// **What every apply does.** It judges the folder (assessFolder), makes a missing one where its data lives, and +// sets the users' entries on the folder itself. What lies under a taken folder is the walks' (ReconcileKept): a +// user added has its entries added to everything under it, a user dropped has exactly its own removed, outside +// the apply and bounded. The watch and the sweep keep what is made later usable (§6, §10). + +// keptRecords is where the engine keeps what it knows of each kept or seen folder: the file beside the state in +// the engine, memory in a test. +type keptRecords interface { + All() (map[string]store.KeptFolder, error) + Update(path string, change func(f *store.KeptFolder, had bool) bool) error +} + +// KeptFolders is the engine's record of the folders it keeps. The engine sets the file beside its state. +var KeptFolders keptRecords = NewKeptMemory() + +// keptMemory is a keptRecords held in memory. +type keptMemory struct { + mu sync.Mutex + m map[string]store.KeptFolder +} + +// NewKeptMemory is an empty record held in memory: a test's, or a one-shot apply's. +func NewKeptMemory() keptRecords { return &keptMemory{m: map[string]store.KeptFolder{}} } + +func (k *keptMemory) All() (map[string]store.KeptFolder, error) { + k.mu.Lock() + defer k.mu.Unlock() + out := map[string]store.KeptFolder{} + for p, f := range k.m { + out[p] = f + } + return out, nil +} + +func (k *keptMemory) Update(path string, change func(f *store.KeptFolder, had bool) bool) error { + k.mu.Lock() + defer k.mu.Unlock() + f, had := k.m[path] + f.Path = path + if change(&f, had) { + k.m[path] = f + } else { + delete(k.m, path) + } + return nil +} + +// keptRecord is the record of path, and whether there is one. A record that cannot be read is none: the folder +// is judged afresh, which waits rather than keeps. +func keptRecord(path string) (store.KeptFolder, bool) { + all, err := KeptFolders.All() + if err != nil { + return store.KeptFolder{}, false + } + f, had := all[path] + return f, had +} + +// folderWant is what every access naming one folder asks of it: the accesses, each uid at the widest use any of +// them gives it (ADR 0325 §4: the tool runner's account stands for every module whose tools use it), and whether +// the controller says it was taken. +type folderWant struct { + path string + ids []string + users map[uint32]store.KeptUser + taken bool +} + +// foldersOf is every declared access's folder, by its clean path. A folder the guard refuses is not read (issue +// 339). +func foldersOf(resources []declaration.Resource, refused map[string]error) map[string]*folderWant { + out := map[string]*folderWant{} + for _, r := range resources { + a, ok := r.(*declaration.Access) + if !ok { + continue + } + path := filepath.Clean(a.Path) + if refused[path] != nil { + continue + } + f := out[path] + if f == nil { + f = &folderWant{path: path, users: map[uint32]store.KeptUser{}} + out[path] = f + } + f.ids = append(f.ids, a.ID) + f.taken = f.taken || a.Taken + for _, u := range a.Users { + had, seen := f.users[uint32(u.UID)] + if seen && (had.Use == declaration.AccessReadWrite || u.Use != declaration.AccessReadWrite) { + continue + } + f.users[uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), Use: u.Use, From: u.From} + } + } + return out +} + +// list is the users, by uid. +func (f *folderWant) list() []store.KeptUser { + if f == nil { + return nil + } + out := make([]store.KeptUser, 0, len(f.users)) + for _, u := range f.users { + out = append(out, u) + } + sort.Slice(out, func(i, j int) bool { return out[i].UID < out[j].UID }) + return out +} + +// usersOf is a list of users as each uid and whether it writes. +func usersOf(us []store.KeptUser) map[uint32]bool { + out := map[uint32]bool{} + for _, u := range us { + out[u.UID] = out[u.UID] || u.Use == declaration.AccessReadWrite + } + return out +} + +// sameUsers is whether two lists give the same uids the same use. +func sameUsers(a, b []store.KeptUser) bool { + x, y := usersOf(a), usersOf(b) + if len(x) != len(y) { + return false + } + for uid, rw := range x { + if w, ok := y[uid]; !ok || w != rw { + return false + } + } + return true +} + +// accessName is the module and its own id for an access, from the resource id `.`. +func accessName(id string) (module, name string) { + if m, ok := moduleOf(id); ok { + return m, id[len(m)+1:] + } + return "", id +} + +// KeepWords and DiskWords are what an access wait says the operator gives (ADR 0325 §7): never a path. +func KeepWords(name string) string { return "let the mesh keep the shared folder " + name } +func DiskWords(name string) string { return "mount the disk that holds " + name } + +// The judgements of a folder. +const ( + // folderKeep is a folder the mesh keeps: taken, or holding no data yet. + folderKeep = "keep" + // folderMake is a folder absent under a parent on its placed filesystem, not root's: made, then kept. + folderMake = "make" + // folderWaitKeep is a folder that holds data and was never taken: it waits for the operator's word. + folderWaitKeep = "wait-keep" + // folderWaitDisk is a folder whose disk is not there. + folderWaitDisk = "wait-disk" + // folderRoot is root's data on its placed filesystem: never kept. + folderRoot = "root" + // folderUnknown is a folder whose filesystem could not be named: left as it is until it is. + folderUnknown = "unknown" +) + +// folderVerdict is the judgement of one folder, with what it read. +type folderVerdict struct { + kind string + // how is why a folder kept is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptMade. + how string + // mount is the mount point that holds what is there. + mount string + // why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link). + why string + err error + // moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh. + moved bool + // unnamed is a folder whose filesystem cannot be named: never kept, nor taken. + unnamed bool +} + +// assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then +// whose it is, then whether the mesh may keep it. Its words carry no path: they reach the operator's channel. +func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict { + others := recordedMounts(path) + info, err := lstatWithin(path) + if errors.Is(err, os.ErrNotExist) { + m, err := NearestAncestor(path) + if err != nil { + return folderVerdict{err: err} + } + if !m.Info.IsDir() { + return folderVerdict{err: errors.New("the folder is not there, and its nearest existing parent is not " + + "a folder (a file or a link); nothing is made under it")} + } + pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others) + if pl.Away { + return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} + } + uid, gid, ok := statOwner(m.Info) + if !ok { + return folderVerdict{err: errors.New("the owner of the folder's nearest existing parent cannot be read " + + "on this machine")} + } + if why := rootOwned(uint32(uid), uint32(gid)); why != "" { + return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " + + "existing parent is " + why} + } + return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade, moved: pl.Moved} + } + if err != nil { + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} + } + if !info.IsDir() { + return folderVerdict{err: errors.New("the folder is not a directory, and an access is a shared directory")} + } + pl := placedOn(ctx, path, path, rec.Mount, others) + if pl.Away { + return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} + } + // Another filesystem at the same mount point (another pool, another disk) is a move too. A record from an earlier + // build names its filesystem another way, and is not compared: the next keep names it this way. + here := PlacedID(pl.Mount) + // **A filesystem that cannot be named now is unknown, never a move** (a name read that did not answer, a + // filesystem with no UUID, id or source): the folder holds what it is — nothing walked, nothing newly kept, + // nothing reset — and is judged again when it is named. One never kept is so never kept: it fails closed. + if here == "" { + return folderVerdict{kind: folderUnknown, mount: pl.Mount, unnamed: true, + why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"} + } + moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here) + facts, err := readFolder(path) + if err != nil { + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} + } + if why := rootOwned(facts.UID, facts.GID); why != "" { + return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} + } + // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a + // folder moved to another is not taken, and asks again (§8). + takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved)) + switch { + case rec.Kept && !moved: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} + case takenHere: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: moved} + case facts.Empty: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: moved} + } + // What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty + // below it included (a move): it waits for the operator's word. + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved} +} + +// movedRecord is f judged afresh on the filesystem now at mount: the warrant's filesystem is kept so the +// controller's word is not read for the new one, and nothing else of what was kept there holds. What the mesh +// added on the old filesystem is left to be taken out once it holds the folder again (Left) only where it was +// walked onto data under a warrant: a folder kept because it was empty, or made, carries entries on that empty +// folder alone, hidden under the mount point and granting nothing on data, and they are forgotten (said once, by +// the apply that saw the move). +func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { + left := f.Left + if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" { + left = addLeft(left, f.Mount, f.Placed, f.Fs, f.Users) + } + placed, fs := placeOf(mount) + return store.KeptFolder{Path: f.Path, Mount: mount, Placed: placed, Fs: fs, Want: f.Want, IDs: f.IDs, + Act: f.Act, TakenOn: f.TakenOn, Left: left} +} + +// placeOf is the name of the filesystem at mount, and what it was made from. +func placeOf(mount string) (placed, fs string) { + fs = filesystemKey(mount) + return placedFrom(mount, fs), fs +} + +// addLeft adds users' entries left on mount, once per mount and uid. +func addLeft(left []store.KeptLeft, mount, placed, fs string, users []store.KeptUser) []store.KeptLeft { + for i := range left { + if left[i].Mount == mount && left[i].Placed == placed { + have := usersOf(left[i].Users) + for _, u := range users { + if _, had := have[u.UID]; !had { + left[i].Users = append(left[i].Users, u) + } + } + return left + } + } + return append(left, store.KeptLeft{Mount: mount, Placed: placed, Fs: fs, Since: time.Now().UTC(), + Users: append([]store.KeptUser(nil), users...)}) +} + +// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the +// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder +// on it is then never kept (it fails closed, and waits). +func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) } + +// placedFrom is PlacedID from a filesystem key already read. +func placedFrom(mount, key string) string { + if mount == "" || key == "" { + return "" + } + sum := sha256.Sum256([]byte(mount + "\x00" + key)) + return "fs-" + hex.EncodeToString(sum[:6]) +} + +// filesystemKey is what names the filesystem at mount across reboots, in this order: +// +// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in +// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first; +// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its +// UUID; +// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own +// source string (both report f_fsid 0). +// +// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of +// these answers within statBound: then nothing is kept there. Read once per mount at a time. +func filesystemKey(mount string) string { + if mount == "" { + return "" + } + v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) }) + if !ok { + return "" + } + return v.(string) +} + +func readFilesystemKey(mount string) string { + if u := MountUUID(mount); u != "" { + return "uuid:" + u + } + if id := MountFsid(mount); strings.Trim(id, "0") != "" { + return "fsid:" + id + } + if source, fstype := mountSourceOf(mount); source != "" && source != "none" { + return "source:" + fstype + ":" + source + } + return "" +} + +// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose +// server is down blocks a stat for ever, and one access must never hold the apply. +var statBound = 5 * time.Second + +// errNoAnswer is a folder whose read did not return within statBound. +var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)") + +// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own. + +func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) } +func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) } + +// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds +// one goroutine per path, never one per apply or look. +type flight struct { + done chan struct{} + v any +} + +var inFlight = struct { + sync.Mutex + reads map[string]*flight +}{reads: map[string]*flight{}} + +// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not +// answer in time. +func boundedOnce(key string, read func() any) (any, bool) { + inFlight.Lock() + f, running := inFlight.reads[key] + if !running { + f = &flight{done: make(chan struct{})} + inFlight.reads[key] = f + go func() { + f.v = read() + inFlight.Lock() + delete(inFlight.reads, key) + inFlight.Unlock() + close(f.done) + }() + } + inFlight.Unlock() + select { + case <-f.done: + return f.v, true + case <-time.After(statBound): + return nil, false + } +} + +func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + type read struct { + info os.FileInfo + err error + } + v, ok := boundedOnce(op+" "+path, func() any { + info, err := stat(path) + return read{info, err} + }) + if !ok { + return nil, errNoAnswer + } + r := v.(read) + return r.info, r.err +} + +// absPath is an absolute path in words: what pathless takes out. +var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`) + +// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other +// path said as "a path". +func Pathless(err error) string { return pathless(err) } + +func pathless(err error) string { + var pe *os.PathError + if errors.As(err, &pe) { + return pe.Op + ": " + pe.Err.Error() + } + return absPath.ReplaceAllString(err.Error(), "${1}a path") +} + +// recordedMounts is every mount point recorded for another kept folder: where a disk was once seen. +func recordedMounts(except string) []string { + all, err := KeptFolders.All() + if err != nil { + return nil + } + var out []string + for p, f := range all { + if p != except && f.Mount != "" && f.Mount != "/" { + out = append(out, f.Mount) + } + } + return out +} + +// RootWords is the reason root's data is stated unhealthy with (ADR 0325 §12): never a path. +func RootWords(why string) string { + return why + ", and root's data is never kept: the operator places the data elsewhere or gives it an owner " + + "(novox/hq ADR 0325 §12)" +} + +// applyAccess is one access, applied (ADR 0325): its folder judged, made when it is missing where its data lives, +// and its users' entries kept on it when the mesh may keep it; waiting otherwise, and failing for root's data. +// Nothing is changed on the folder but the entries the mesh adds and the mask that shows them. +func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, + in inputs) (Outcome, error) { + // What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5). + gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) + out, err := applyAccessFolder(ctx, r, in) + if len(left) > 0 { + out.user, out.groups = previous.User, left + } + if said != "" { + out.Detail = joinDetail(out.Detail, said) + if gave && out.Action == "unchanged" { + // Updated, so what names the access under restart-on — the tool runner — holds the groups it has now. + out.Action = "updated" + } + } + return out, err +} + +// applyAccessFolder is applyAccess's folder. +func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (Outcome, error) { + out := begin(r) + path := filepath.Clean(r.Path) + want := in.folders[path] + if !keeps(want, in.identities) { + return legacyAccess(r) + } + users := want.list() + _, name := accessName(r.ID) + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, want != nil && want.taken) + if v.err != nil { + return out, v.err + } + if v.moved { + // What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty + // below it): judged afresh, the record of what was kept there no longer holds. + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + *f = movedRecord(*f, v.mount) + return had + }) + } + note := "" + if v.moved { + note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh" + if rec.How != store.KeptByWarrant { + note += "; the entries on the empty folder kept below are forgotten: hidden under the mount, they grant " + + "nothing on data" + } + } + if len(r.Grant) > 0 { + note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" + } + switch v.kind { + case folderUnknown: + out.Action = "waiting" + out.Detail = v.why + ": the folder is left exactly as it is, and judged again on the next apply" + note + return out, nil + case folderWaitDisk: + out.Action = "waiting" + out.Detail = "the disk that holds it is not there: nothing is made, and nothing mounts it until it is " + + "(novox/hq ADR 0325 §9); waits for the operator to " + DiskWords(name) + note + return out, nil + case folderRoot: + if len(users) == 0 { + out.Action, out.Detail = "unchanged", "present or not, nothing is kept: no identity uses it"+note + return out, nil + } + return out, errors.New(RootWords(v.why) + "; nothing was made or changed") + case folderMake: + made, err := CreatePath(path) + if err != nil { + return out, err + } + out.Action = "created" + out.Detail = fmt.Sprintf("made %d level(s) as its nearest existing parent is: owner, group and mode "+ + "(novox/hq ADR 0325 §9)", made) + if len(users) == 0 { + out.Detail += note + return out, nil + } + case folderWaitKeep: + if len(users) == 0 { + out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"+note + return out, nil + } + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.Want, f.IDs = v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) + return true + }) + out.Action = "waiting" + out.Detail = "it holds data the mesh has not taken: nothing is changed on it until the operator's warrant " + + "(novox/hq ADR 0325 §7, §8); waits for the operator to " + KeepWords(name) + note + return out, nil + } + if len(users) == 0 { + // Nobody uses it any more: what the mesh added goes on the next walk. + dropping := false + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Want = nil + dropping = len(f.Users) > 0 + return had + }) + if out.Action == "" { + out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it" + } + if dropping { + out.Detail += "; the entries the mesh added are taken out by the next walk" + } + out.Detail += note + return out, nil + } + changed, err := keepTop(path, usersOf(users)) + if errors.Is(err, ErrNoACL) { + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.Want = v.mount, users + f.Placed, f.Fs = placeOf(v.mount) + return true + }) + out.Action = "waiting" + out.Detail = "its filesystem allows no access entries here: the take act lets it, on the operator's word " + + "(novox/hq ADR 0325 §8); waits for the operator to " + KeepWords(name) + note + return out, nil + } + if err != nil { + return out, err + } + var pending string + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + first := !f.Kept + f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) + if v.how == store.KeptByWarrant && f.TakenOn == "" { + // The controller says it was taken, and this engine has no record of where: here, now. + f.TakenOn = PlacedID(v.mount) + } + if first { + f.How, f.At = v.how, time.Now().UTC() + // A folder that held nothing has nothing under it to walk; one the controller says was taken + // without this engine's record is walked whole. + if v.how == store.KeptEmpty || v.how == store.KeptMade { + f.Users = users + } + } + if !sameUsers(f.Users, f.Want) { + pending = "; the entries under it follow on the next walk" + } + return true + }) + if out.Action == "" { + out.Action = "unchanged" + if changed { + out.Action = "updated" + } + } + out.Detail = joinDetail(out.Detail, "kept for "+usersWords(users)+": its own entries and default, nothing "+ + "else of it changed (novox/hq ADR 0325 §5)"+pending+note) + return out, nil +} + +// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes +// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group. +// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next +// apply tries again — and what was said; nothing for an access that granted nothing. +func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted, + handed Handed) (bool, []string, string) { + if previous.User == "" || len(previous.Groups) == 0 { + return false, nil, "" + } + account := previous.User + in, err := system.GroupsOf(ctx, system.Runner(run), account) + if err != nil { + return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+ + "back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err) + } + already := setOf(in) + gave := false + var left, said []string + for _, g := range previous.Groups { + switch { + case !already[g]: + continue + case len(wanted.othersAsk(account, g, previous.ID)) > 0: + said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g, + strings.Join(wanted.othersAsk(account, g, previous.ID), ", "))) + handed.hand(account, g) + continue + } + gone, why := leaveGroup(ctx, sys, account, g, run) + if !gone { + left = append(left, g) + why += "; tried again on the next apply" + } + gave = gave || gone + said = append(said, why) + } + return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ") +} + +// usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)". +func usersWords(users []store.KeptUser) string { + words := make([]string, 0, len(users)) + for _, u := range users { + who := u.Identity + if u.From == declaration.FromToolRunner { + who = "the tool runner's " + who + } + words = append(words, who+" ("+u.Use+")") + } + return strings.Join(words, ", ") +} + +// forgetUndeclared marks every recorded folder no declared access names any more: nobody wants its entries, and +// the next walk takes out exactly what the mesh added, then forgets it. +func forgetUndeclared(folders map[string]*folderWant) { + all, err := KeptFolders.All() + if err != nil { + return + } + for path := range all { + if folders[path] != nil { + continue + } + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Want = nil + // A folder never kept, or kept with nobody's entries left anywhere, is forgotten now. + return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0 + }) + } +} + +// MissingPath is an accessed folder that is not there, and its nearest existing ancestor. +type MissingPath struct { + Ancestor string + // Levels are the names to make below Ancestor, outermost first. + Levels []string + // Info is the ancestor as lstat reads it: a link is not a directory. + Info os.FileInfo +} + +// NearestAncestor reads, for a path that is not there, its nearest existing ancestor by lstat. An error for a +// path that is there, one not stated exactly, or an ancestor that cannot be read. +func NearestAncestor(path string) (MissingPath, error) { + if err := keptPath(path); err != nil { + return MissingPath{}, err + } + if _, err := lstatWithin(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return MissingPath{}, errors.New("the folder is there, or cannot be read") + } + var levels []string + for p := path; p != "/"; p = filepath.Dir(p) { + levels = append([]string{filepath.Base(p)}, levels...) + parent := filepath.Dir(p) + info, err := lstatWithin(parent) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return MissingPath{}, errors.New("a parent of the folder cannot be read: " + pathless(err)) + } + return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil + } + return MissingPath{}, errors.New("no part of the folder's path is there") +} + +// accessPaths is every declared access's folder with its id: what no container mounts while it is not there. +// +// Only accesses the mesh keeps (Keeps): an older controller's access holds no container, as before ADR 0325. +func accessPaths(resources []declaration.Resource, refused map[string]error, identities bool) map[string]string { + folders := foldersOf(resources, refused) + out := map[string]string{} + for _, r := range resources { + a, ok := r.(*declaration.Access) + if !ok { + continue + } + if path := filepath.Clean(a.Path); refused[path] == nil && keeps(folders[path], identities) { + out[path] = a.ID + } + } + return out +} + +// keeps is whether ADR 0325's keeping applies to a folder: some access naming it carries users, or the +// declaration carries identities — a controller that speaks link.IdentityContract. An older controller's access +// (no users, no identities) is applied exactly as before: mounted when present, refused when absent, nothing kept, +// nothing made, nothing judged (ADR 0051 §7). +func keeps(f *folderWant, identities bool) bool { + return identities || (f != nil && len(f.users) > 0) +} + +// Keeps is keeps for a whole declaration's access at path, for the engine's looks. +func Keeps(d *declaration.Declaration, path string) bool { + if d == nil { + return false + } + return keeps(foldersOf(d.Resources, nil)[filepath.Clean(path)], len(d.Identities) > 0) +} + +// mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there +// now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026). +func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string { + for _, v := range c.Volumes { + src := mountSource(v) + if !strings.HasPrefix(src, "/") { + continue + } + for path, id := range accesses { + if !atOrBelow(filepath.Clean(src), path) { + continue + } + if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) { + return id + } + // There, and its disk not: what is at the path is the empty mount point on the filesystem below, and + // a container's writes would land there (§9). + rec, _ := keptRecord(path) + if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away { + return id + } + } + } + return "" +} + +// WaitingOnAccess is every container of the declaration that waits for an accessed folder not there, read now: +// not created by the apply, so not judged alive or dead — the access says why. +func WaitingOnAccess(d *declaration.Declaration) map[string]bool { + out := map[string]bool{} + if d == nil { + return out + } + accesses := accessPaths(d.Resources, refusedPlaces(d.Resources), len(d.Identities) > 0) + if len(accesses) == 0 { + return out + } + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" { + out[c.ID] = true + } + } + return out +} + +// The judgements of an access as a look says them. +const ( + VerdictKeep = "keep" + VerdictMake = "make" + VerdictWaitKeep = "wait-keep" + VerdictWaitDisk = "wait-disk" + VerdictRoot = "root" + VerdictBroken = "broken" + // VerdictUnknown is a folder whose filesystem could not be named this look: neither kept nor faulted. + VerdictUnknown = "unknown" + // VerdictNobody is a folder no identity uses: nothing is kept, and only a missing disk is said. + VerdictNobody = "nobody" +) + +// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at +// it for a wait, with no path (PlacedID). +type AccessJudgement struct { + Kind, Why, Placed string +} + +// AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the +// controller says it was taken, used whether any identity uses it. A folder the mesh could keep that no apply has +// kept — its filesystem allowing no ACLs — waits as one not taken does. +func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJudgement { + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, taken) + switch { + case v.err != nil: + return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)} + case v.kind == folderWaitDisk: + return AccessJudgement{Kind: VerdictWaitDisk} + case v.kind == folderUnknown: + return AccessJudgement{Kind: VerdictUnknown, Why: v.why} + case !used: + return AccessJudgement{Kind: VerdictNobody} + case v.kind == folderRoot: + return AccessJudgement{Kind: VerdictRoot, Why: v.why} + case v.kind == folderMake: + return AccessJudgement{Kind: VerdictMake} + case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken: + return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)} + } + return AccessJudgement{Kind: VerdictKeep} +} + +// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder, +// why, in words with no path. +type Leftover struct { + Module, Access, Name, Path, Why string +} + +// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names +// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each +// is said until its drop walk has run. +func Leftovers() []Leftover { + all, err := KeptFolders.All() + if err != nil { + return nil + } + var out []Leftover + for path, f := range all { + module, name, access := recordName(path, f) + dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting" + switch { + case dropping: + // Why the walk could not run, as it recorded it: its disk not there, or its filesystem not named. + why := strings.TrimSuffix(strings.TrimPrefix(f.Act.Why, "not walked: "), "; walked once it is") + if why == "" { + why = "the walk could not run" + } + out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + + name + " could not be removed: " + why + "; they are removed once it can (novox/hq ADR 0325 §8)"}) + case len(f.Left) > 0: + out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + + name + " on a filesystem no longer at its folder could not be removed: they are removed once that " + + "filesystem is back (novox/hq ADR 0325 §8)"}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access }) + return out +} + +// recordName is the module, the name and the access a kept folder's record is said under: the accesses that last +// named it, or for a record written before it kept them, a name with no path — the module of a container identity +// among its users, and a folder named by a digest of its path. +func recordName(path string, f store.KeptFolder) (module, name, access string) { + if len(f.IDs) > 0 { + module, name = accessName(f.IDs[0]) + return module, name, f.IDs[0] + } + sum := sha256.Sum256([]byte(path)) + name = "folder-" + hex.EncodeToString(sum[:4]) + for _, u := range append(append([]store.KeptUser(nil), f.Users...), f.Want...) { + if m, ok := strings.CutPrefix(u.Identity, declaration.IdentityPrefix); ok && u.From == declaration.FromContainer { + return m, name, m + "." + name + } + } + return "", name, name +} + +// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7). +// +// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container +// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026 +// records. So the host checks first and says plainly that the operator must provide the path, rather than +// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports +// a change: the machine did not move, the host merely confirmed a fact about it. +func legacyAccess(r *declaration.Access) (Outcome, error) { + out := begin(r) + info, err := os.Stat(r.Path) + if errors.Is(err, os.ErrNotExist) { + return out, fmt.Errorf( + "%s is not there, and the mesh does not own it — the operator must provide it. It is "+ + "shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+ + "nothing, so a missing one is said here rather than made as root by the container "+ + "runtime", r.Path) + } + if err != nil { + return out, err + } + if !info.IsDir() { + return out, fmt.Errorf( + "%s is not a directory, and an access is a shared directory the operator provides", + r.Path) + } + mode := declaration.AccessRead + if r.Mode != "" { + mode = r.Mode + } + out.Action = "unchanged" + out.Detail = "operator-owned; present, " + mode + ", nothing managed" + return out, nil +} diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go new file mode 100644 index 0000000..a4f6afc --- /dev/null +++ b/internal/apply/keeper_linux.go @@ -0,0 +1,446 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The walks under a kept folder, beside the apply (novox/hq ADR 0325 §6, §8, §10). +// +// The take act adds the users' entries to everything under a folder on the operator's warrant; ReconcileKept adds a +// joining user's and removes exactly a dropped user's under a folder already taken; the sweep looks at a bounded +// part of each kept folder on every look, going on where the last stopped, and puts back what an app narrowed. +// One walk at a time per folder: a sweep finds a folder being walked and waits for the next look. + +// folderLocks is one lock per kept folder, held by every walk of it. +var folderLocks sync.Map + +func folderLock(path string) *sync.Mutex { + l, _ := folderLocks.LoadOrStore(path, &sync.Mutex{}) + return l.(*sync.Mutex) +} + +// actRecord is a walk's counts as the record keeps them. +func actRecord(act, warrant, by string, started time.Time, w *aclWalk, err error) *store.KeptAct { + c := w.counts + rec := &store.KeptAct{Act: act, Warrant: warrant, By: by, State: "done", Started: started, Ended: time.Now().UTC(), + Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other, Mounts: c.Mounts} + switch { + case errors.Is(err, ErrWalkBound): + rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on at the next apply" + case err != nil: + rec.State, rec.Why = "failed", err.Error() + case w.Failed != "": + rec.Why = fmt.Sprintf("%d entries could not be given their entries: %s", c.Unrepaired, w.Failed) + } + return rec +} + +// ReconcileKept walks every kept folder whose entries under it are not what the last apply declared: a user that +// joined has its entries added to everything under it, a dropped user — or one whose use changed — has exactly its +// own taken out (ADR 0325 §8). Bounded by ActLimits; a walk stopped is gone on with at the next call. A folder +// nobody uses any more is forgotten once its entries are out. What it did, in words, for the engine to say. +func ReconcileKept(ctx context.Context) []string { + all, err := KeptFolders.All() + if err != nil { + return []string{"the kept folders could not be read: " + err.Error()} + } + var said []string + for path, f := range all { + if ctx.Err() != nil { + break + } + if len(f.Left) > 0 { + said = append(said, dropLeft(path, f)...) + if f, _ = keptRecord(path); f.Path == "" { + continue + } + } + if !f.Kept || sameUsers(f.Users, f.Want) { + continue + } + said = append(said, reconcileOne(ctx, path, f)...) + } + return said +} + +// LeftFor is how long entries left on a filesystem are kept recorded, and said, while it is not back. +var LeftFor = 30 * 24 * time.Hour + +// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds +// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they +// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and +// said (Leftovers). +func dropLeft(path string, f store.KeptFolder) []string { + lock := folderLock(path) + lock.Lock() + defer lock.Unlock() + holding := "" + if _, err := lstatWithin(path); err == nil { + holding = mountHolding(path, MountPoints()) + } + holdingID := "" + if holding != "" { + holdingID = PlacedID(holding) + } + var said []string + var still []store.KeptLeft + var mine []store.KeptUser + for _, l := range f.Left { + // On the folder's own filesystem now: the same name, or a name of an earlier build's form at the same + // mount point (which cannot be compared, and is never walked away). + same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed || l.Fs == "") + back := l.Mount == holding && (l.Placed == "" || l.Placed == holdingID) + if l.Since.IsZero() { + l.Since = time.Now().UTC() + } + if !same && !back && time.Since(l.Since) > LeftFor { + // **A filesystem not back within LeftFor is gone** (a disk replaced, a pool destroyed): what was left + // on it is forgotten, said once, rather than said unhealthy for ever. + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem not back for %s "+ + "are forgotten: it is taken for gone", path, LeftFor)) + continue + } + switch { + case f.Kept && same: + // Left on the filesystem the folder is kept on now: the folder's own entries. + mine = append(mine, l.Users...) + case same: + // On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's + // once it is kept there again. + still = append(still, l) + case !back: + still = append(still, l) + default: + drop := map[uint32]bool{} + for _, u := range l.Users { + drop[u.UID] = true + } + w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop)) + if err != nil || w.Failed != "" { + still = append(still, l) + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed))) + continue + } + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files)) + } + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Left = still + for _, u := range mine { + if _, has := usersOf(k.Users)[u.UID]; !has { + k.Users = append(k.Users, u) + } + } + return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0) + }) + return said +} + +func pathlessOr(err error) string { + if err == nil { + return "" + } + return pathless(err) +} + +func firstOf(a, b string) string { + if a != "" { + return a + } + return b +} + +func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string { + lock := folderLock(path) + lock.Lock() + defer lock.Unlock() + // **Judged as an apply judges it, before every walk** (§9, §12): a disk not there leaves its mount point — + // an empty folder of root's on the filesystem below — at the path, and a walk there would write onto root's + // folder, or "drop" entries from a folder that never had them and forget the ones on the data. Nothing is + // walked then, and the record stays for the walk that can reach the data. + if v := assessFolder(ctx, path, f, false); v.err != nil || v.kind != folderKeep || v.moved { + why := "the disk that holds it is not there" + switch { + case v.err != nil: + why = pathless(v.err) + case v.kind == folderRoot: + why = RootWords(v.why) + case v.kind == folderUnknown: + why = v.why + case v.moved || v.kind == folderWaitKeep: + why = "what is at it now is not what the mesh kept there" + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Act = &store.KeptAct{Act: "walk", State: "waiting", Started: time.Now().UTC(), + Why: "not walked: " + why + "; walked once it is"} + return had + }) + return []string{fmt.Sprintf("the kept folder %s: not walked, %s", path, why)} + } + have, want := usersOf(f.Users), usersOf(f.Want) + drop := map[uint32]bool{} + adds := false + for uid, rw := range have { + if w, ok := want[uid]; !ok || w != rw { + drop[uid] = true + } + } + for uid, rw := range want { + if h, ok := have[uid]; !ok || h != rw { + adds = true + } + } + act, visit := "add", keepVisit(want) + switch { + case len(drop) > 0 && adds: + visit = dropThenKeep(drop, want) + case len(drop) > 0: + act, visit = "drop", dropVisit(drop) + } + started := time.Now().UTC() + w, err := walkKept(path, ActLimits, machineStats, nil, visit) + rec := actRecord(act, "", "", started, w, err) + _ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool { + k.Act = rec + // Done only when every entry was reached and changed: one that failed is walked again. + if err == nil && w.Failed == "" { + k.Users = f.Want + } + return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0) + }) + return []string{fmt.Sprintf("the kept folder %s: %s walk %s (%d folders and %d files changed, %d had "+ + "them)%s", path, act, rec.State, rec.Folders, rec.Files, rec.Kept, orWhy(rec.Why))} +} + +func orWhy(why string) string { + if why == "" { + return "" + } + return ": " + why +} + +// TakeFolder is the take act (ADR 0325 §8): on the operator's warrant, the users' entries added to the folder of +// the access id and everything under it. It judges the ask against the declaration last applied and the folder as +// it is, sets the filesystem's ACL type where a ZFS dataset allows none yet, and starts the walk: what it answers +// at once, and a channel closed when the walk is through. Refused, nothing is changed. +func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warrant, by string, + run Runner) (<-chan struct{}, string, error) { + if d == nil { + return nil, "", errors.New("this engine has applied no declaration, and takes no folder it was not told of") + } + var access *declaration.Access + for _, r := range d.Resources { + if a, ok := r.(*declaration.Access); ok && a.ID == id { + access = a + } + } + switch { + case access == nil: + return nil, "", fmt.Errorf("the access %s is not declared on this machine", id) + case filepath.Clean(access.Path) != path: + return nil, "", fmt.Errorf("the access %s is declared at another folder than the one asked", id) + } + folders := foldersOf(d.Resources, refusedPlaces(d.Resources)) + want := folders[path] + users := want.list() + if len(users) == 0 { + return nil, "", errors.New("no identity uses the folder, so there is nothing to keep") + } + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, false) + switch { + case v.err != nil: + return nil, "", v.err + case v.kind == folderWaitDisk: + return nil, "", errors.New("the disk that holds the folder is not there") + case v.kind == folderRoot: + return nil, "", errors.New(RootWords(v.why)) + case v.kind == folderMake: + return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody") + case v.unnamed: + return nil, "", errors.New(v.why + ", so a warrant could name no filesystem it was given for") + } + lock := folderLock(path) + if !lock.TryLock() { + return nil, "", errors.New("a walk of the folder is running; asked again once it is through, the act goes on") + } + said := "" + _, err := keepTop(path, usersOf(users)) + if errors.Is(err, ErrNoACL) { + allowed, aerr := allowACLs(ctx, v.mount, run) + if aerr != nil { + lock.Unlock() + return nil, "", aerr + } + said = allowed + "; " + _, err = keepTop(path, usersOf(users)) + } + if err != nil { + lock.Unlock() + return nil, "", err + } + started := time.Now().UTC() + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + if v.moved { + *f = movedRecord(*f, v.mount) + } + if !f.Kept { + f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil + } + f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids + // The warrant is for the filesystem it was carried out on (§8), named as a wait names it. + f.Placed, f.Fs = placeOf(v.mount) + f.TakenOn = f.Placed + f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} + return true + }) + done := make(chan struct{}) + go func() { + defer close(done) + defer lock.Unlock() + w, err := walkKept(path, ActLimits, machineStats, nil, keepVisit(usersOf(users))) + rec := actRecord("take", warrant, by, started, w, err) + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Act = rec + if err == nil { + f.Users = users + } + return true + }) + }() + return done, said + "the take act started under the warrant " + warrant + ": the entries of " + + usersWords(users) + " are added to everything under the folder, following no link and entering no other " + + "filesystem", nil +} + +// allowACLs lets the filesystem at mount carry ACLs, inside the take act only (ADR 0325 §8): a ZFS dataset's +// acltype set to posixacl, the dataset read from the machine's mounts. Any other filesystem is refused, naming why. +func allowACLs(ctx context.Context, mount string, run Runner) (string, error) { + source, fstype := mountSourceOf(mount) + if fstype != "zfs" || source == "" { + return "", fmt.Errorf("the filesystem that holds the folder (%s) allows no access entries, and only a ZFS "+ + "dataset's can be let by the act; the operator mounts it with ACLs", orUnknown(fstype)) + } + if _, err := run(ctx, "zfs", "set", "acltype=posixacl", source); err != nil { + return "", fmt.Errorf("the ZFS dataset that holds the folder could not be let carry access entries "+ + "(zfs set acltype=posixacl): %w", err) + } + return "set acltype=posixacl on the ZFS dataset " + source, nil +} + +func orUnknown(s string) string { + if s == "" { + return "of a type the machine does not say" + } + return s +} + +// KeptLook is what one look found of a kept folder (ADR 0325 §10). +type KeptLook struct { + // Short is every user the folder itself gives less than its right, by entry and mask together, in words. + Short []string + // Busy is a walk of the folder running: the sweep waited. + Busy bool + // Sweep is the sweep's place after this look. + Sweep store.KeptSweep + // Repaired is how many entries this look found narrowed and put back; Unrepaired how many it could not. + Repaired, Unrepaired int + // Failed is why the sweep could not look, in words with no path. + Failed string +} + +// LookKept looks at a kept folder once: the folder itself judged for each user, then the sweep's bounded part of +// what is under it, from where the last look stopped, putting back what an app narrowed. The sweep's place is +// recorded; a look that ran out of its bound says so. +func LookKept(path string, users []store.KeptUser, limits WalkLimits) KeptLook { + var out KeptLook + short, err := judgeTop(path, users) + if err != nil { + out.Failed = err.Error() + return out + } + out.Short = short + lock := folderLock(path) + if !lock.TryLock() { + out.Busy = true + rec, _ := keptRecord(path) + out.Sweep = rec.Sweep + return out + } + defer lock.Unlock() + rec, _ := keptRecord(path) + sw := rec.Sweep + w, err := walkKept(path, limits, machineStats, sw.Cursor, keepVisit(usersOf(users))) + out.Repaired, out.Unrepaired = w.counts.Narrowed-w.counts.Unrepaired, w.counts.Unrepaired + sw.Read += w.counts.Read + sw.Narrowed += w.counts.Narrowed + sw.Unrepaired += w.counts.Unrepaired + switch { + case errors.Is(err, ErrWalkBound): + sw.Cursor, sw.Bound = w.Cursor, true + case err != nil: + out.Failed = err.Error() + default: + // Covered whole: the next look starts a new pass, and what this one could not put back is kept until + // the next pass is through. + sw = store.KeptSweep{Covered: time.Now().UTC(), LastUnrepaired: sw.Unrepaired} + } + out.Sweep = sw + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Sweep = sw + return had + }) + return out +} + +// judgeTop is every user the folder itself gives less than its right, read only. +func judgeTop(path string, users []store.KeptUser) ([]string, error) { + var short []string + _, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return nil, err + } + defer unix.Close(fd) + a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + return nil, err + } + for _, u := range users { + need := needOn(u.Use == declaration.AccessReadWrite, true) + if got := effective(a, u.UID, st.Uid, uint32(st.Mode)&0o7777); got&need != need { + short = append(short, fmt.Sprintf("%s has %s of the %s it needs", u.Identity, permWords(got), + permWords(need))) + } + } + return short, nil +} + +// JudgeKept reads, without changing anything, how many entries under a kept folder give a user less than its +// right, within limits: what the access says while an entry an app narrowed is not put back yet (ADR 0325 §6). +func JudgeKept(path string, users []store.KeptUser, limits WalkLimits) (WalkCounts, error) { + um := usersOf(users) + w, err := walkKept(path, limits, machineStats, nil, func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, + error) { + a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + return false, false, false, err + } + n, u := judgeACL(a, um, st, folder) + return false, n, u, nil + }) + return w.counts, err +} diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go new file mode 100644 index 0000000..107f8ad --- /dev/null +++ b/internal/apply/kept_test.go @@ -0,0 +1,1523 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/store" +) + +// The engine's half of ADR 0325's checks (novox/hq ADR 0325, "How it is checked", issue 567): a shared folder kept +// through access entries, on a filesystem of the test's own. Nothing here needs root: a file's owner may set its +// ACL, and the identities are numbers no account here has. + +const ( + radarr = 61001 // a container's identity, read-write + plex = 61002 // a container's identity, read + runner = 61003 // the tool runner's account +) + +// aclFolder is a folder of the test's own on a filesystem that carries ACLs, or the test is skipped saying why. +func aclFolder(t *testing.T) string { + t.Helper() + dir := userDir(t) + fd, err := unix.Open(dir, unix.O_RDONLY|unix.O_DIRECTORY, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + probe := keptACL(minimalACL(0o755), map[uint32]uint16{radarr: 5}) + if err := setACL(fd, aclAccessXattr, probe); errors.Is(err, ErrNoACL) { + t.Skipf("the test's filesystem (%s) carries no POSIX ACLs (ENOTSUP), and every check here reads them", dir) + } else if err != nil { + t.Fatal(err) + } + if err := unix.Fremovexattr(fd, aclAccessXattr); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + return dir +} + +// keeping has the test keep its own record of kept folders, and judge mounts as the test says. +func keeping(t *testing.T, points []string, targets []string) { + t.Helper() + was, wasPoints, wasTargets := KeptFolders, MountPoints, MountTargets + KeptFolders = NewKeptMemory() + forgetMountHistory() + MountPoints = func() []string { return append([]string{"/"}, points...) } + MountTargets = func(context.Context) []string { return targets } + t.Cleanup(func() { + KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets + forgetMountHistory() + }) +} + +// accessJSON is an access of module to path with users, each "identity:uid:use:from". +func accessJSON(id, path, mode string, taken bool, users ...string) string { + var us []string + for _, u := range users { + f := strings.Split(u, ":") + us = append(us, fmt.Sprintf(`{"identity":%q,"uid":%s,"use":%q,"from":%q}`, f[0], f[1], f[2], f[3])) + } + return fmt.Sprintf(`{"id":%q,"type":"access","path":%q,"mode":%q,"taken":%v,"users":[%s]}`, id, path, mode, + taken, strings.Join(us, ",")) +} + +// readACLs is a path's access ACL and default ACL (nil when it has none), and its stat. +func readACLs(t *testing.T, path string) (acl, acl, unix.Stat_t) { + t.Helper() + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NOFOLLOW, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + t.Fatal(err) + } + access, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + t.Fatal(err) + } + def, err := getACL(fd, aclDefaultXattr, uint32(st.Mode)&0o7777) + if err != nil { + t.Fatal(err) + } + return access, def, st +} + +// effectiveOn is what uid may do on path, by entry and mask together. +func effectiveOn(t *testing.T, path string, uid uint32) uint16 { + t.Helper() + a, _, st := readACLs(t, path) + return effective(a, uid, st.Uid, uint32(st.Mode)&0o7777) +} + +func applyAccesses(t *testing.T, known store.State, resources ...string) (Report, store.State, error) { + t.Helper() + toUser(t) + d := declare(t, strings.Join(resources, ",")) + toUser(t) + return Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, noServices, nil, nil) +} + +// Row 5: entries and default entries per identity and use; owner, owning group, owner's and others' bits unchanged; +// the owning group's entry equals its former group bits; the mask equals what the entries need; an entry the mesh +// did not add is untouched. +func TestAKeptFolderCarriesAnEntryPerUserAndNothingElseOfItChanges(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "movies") + if err := os.Mkdir(lib, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lib, 0o750); err != nil { + t.Fatal(err) + } + // The operator's own entry, there before: never the mesh's to change. + fd, _ := unix.Open(lib, unix.O_RDONLY|unix.O_DIRECTORY, 0) + operator := keptACL(minimalACL(0o750), map[uint32]uint16{60099: permRead}) + operator[len(operator)-1].Perm = permRead | permExec // its mask: r-x, as the operator set it + if err := setACL(fd, aclAccessXattr, operator); err != nil { + t.Fatal(err) + } + unix.Close(fd) + toUser(t) + _, _, before := readACLs(t, lib) + + report, _, err := applyAccesses(t, store.State{}, + accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container"), + accessJSON("plex.movies", lib, "read", false, "mesh-plex:61002:read:container")) + if err != nil { + t.Fatalf("an empty folder was not kept: %v", err) + } + access, def, after := readACLs(t, lib) + if bits, _ := access.perm(aclUser, radarr); bits != 7 { + t.Errorf("the read-write identity's entry is %s, and it needs rwx", permWords(bits)) + } + if bits, _ := access.perm(aclUser, plex); bits != 5 { + t.Errorf("the read identity's entry is %s, and it needs r-x", permWords(bits)) + } + for _, uid := range []uint32{radarr, plex} { + if _, ok := def.perm(aclUser, uid); !ok { + t.Errorf("uid %d has no default entry: what is made under the folder would not be its", uid) + } + } + if bits, _ := access.perm(aclUser, 60099); bits != permRead { + t.Errorf("the operator's entry was changed to %s", permWords(bits)) + } + if g, _ := access.perm(aclGroupObj, 0); g != 5 { + t.Errorf("the owning group's entry is %s, and its group bits gave r-x", permWords(g)) + } + if m, _ := access.perm(aclMask, 0); m != 7 { + t.Errorf("the mask is %s, and the entries need rwx", permWords(m)) + } + if after.Uid != before.Uid || after.Gid != before.Gid { + t.Error("the folder's owner or owning group changed") + } + if after.Mode&0o707 != before.Mode&0o707 { + t.Errorf("the owner's or others' bits changed: %o to %o", before.Mode&0o777, after.Mode&0o777) + } + if after.Mode&0o070 != 0o070 { + t.Errorf("the group bits show %o, and they show the mask the entries need (rwx)", (after.Mode>>3)&7) + } + rec, _ := keptRecord(lib) + if !rec.Kept || rec.How != store.KeptEmpty { + t.Errorf("the folder was not recorded kept as one that held nothing: %+v", rec) + } + for _, o := range report.Outcomes { + if o.ID == "radarr.movies" && o.Action == "waiting" { + t.Errorf("an empty folder waited for a warrant: %s", o.Detail) + } + } +} + +// Row 4: the runner's entry is the widest use among the modules whose tools use the folder. +func TestTheToolRunnersEntryIsTheWidestUseAnyModuleGivesIt(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "books") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + _, _, err := applyAccesses(t, store.State{}, + accessJSON("reader.books", lib, "read", false, "nox:61003:read:tool-runner"), + accessJSON("importer.books", lib, "read-write", false, "nox:61003:read-write:tool-runner")) + if err != nil { + t.Fatal(err) + } + if got := effectiveOn(t, lib, runner); got != 7 { + t.Fatalf("the tool runner may %s, and one module whose tools use it declared read-write", permWords(got)) + } +} + +// Row 6: a file made at 0666 by one identity is writable by another read-write identity at once; one made at 0644 +// is writable by it after the watch's repair, and before it the judgement says so; with watches exhausted, the +// sweep repairs it; the judgement reads the effective right (entry and mask). +func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "downloads") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + users := []string{"mesh-radarr:61001:read-write:container", "mesh-sab:61004:read-write:container"} + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.downloads", lib, "read-write", false, + users...)); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + + usual := filepath.Join(lib, "usual") + if err := os.WriteFile(usual, nil, 0o666); err != nil { + t.Fatal(err) + } + if got := effectiveOn(t, usual, radarr); got&6 != 6 { + t.Fatalf("a file made at 0666 gives another read-write identity %s at once", permWords(got)) + } + + w, err := NewWatcher() + if err != nil { + t.Skipf("no inotify here: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + defer func() { cancel(); <-ended }() + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + + narrow := filepath.Join(lib, "narrow") + f, err := os.OpenFile(narrow, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.Close() + waitFor(t, "the watch widened the mask of a file made at 0644", func() bool { + return effectiveOn(t, narrow, radarr)&6 == 6 + }) + + // Before a repair the judgement says so: a mask narrowed by hand where nothing watches. + w.Set(nil) + if err := os.Chmod(usual, 0o644); err != nil { + t.Fatal(err) + } + toUser(t) + c, err := JudgeKept(lib, rec.Want, ActLimits) + if err != nil { + t.Fatal(err) + } + if c.Narrowed == 0 { + t.Fatal("a file whose mask gives a read-write user only read is not said by the judgement") + } + + // Watches run out: the record says so, and the sweep puts it back. + was := MaxWatchedPerFolder + MaxWatchedPerFolder = 0 + defer func() { MaxWatchedPerFolder = was }() + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + if r, _ := keptRecord(lib); !r.OutOfWatches { + t.Error("watches that ran out are not recorded") + } + toUser(t) + look := LookKept(lib, rec.Want, SweepLimits) + if look.Repaired == 0 { + t.Errorf("the sweep put nothing back: %+v", look) + } + if got := effectiveOn(t, usual, radarr); got&6 != 6 { + t.Fatalf("after the sweep a read-write identity may %s", permWords(got)) + } + toUser(t) + if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { + t.Fatalf("after the sweep %d entries still give a user less than its right", c.Narrowed) + } +} + +func waitFor(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatalf("not within 3s: %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +// Row 8, the engine's part: a folder that holds data waits, and nothing is changed without a warrant; the act adds +// the entries under the folder, follows no link, enters no other filesystem; a dropped access removes exactly its +// own entries. +func TestAFolderThatHoldsDataIsTakenOnlyOnAWarrantAndLetGoEntryByEntry(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "music") + outside := filepath.Join(dir, "outside") + for _, d := range []string{lib, filepath.Join(lib, "album"), outside} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + song := filepath.Join(lib, "album", "song") + if err := os.WriteFile(song, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(lib, "link")); err != nil { + t.Fatal(err) + } + a := accessJSON("lidarr.music", lib, "read-write", false, "mesh-lidarr:61001:read-write:container", + "mesh-plex:61002:read:container") + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "lidarr.music"); o.Action != "waiting" || !strings.Contains(o.Detail, KeepWords("music")) { + t.Fatalf("a folder that holds data did not wait for the operator: %+v", o) + } + if has := effectiveOn(t, lib, radarr); has != 0 { + t.Fatal("the folder was given an entry without a warrant") + } + if strings.Contains(KeepWords("music"), "/") || strings.Contains(DiskWords("music"), "/") { + t.Fatal("a wait's words name a path") + } + + d := declare(t, a) + toUser(t) + done, said, err := TakeFolder(context.Background(), d, "lidarr.music", lib, "warrant-7", "the operator", noServices) + if err != nil { + t.Fatalf("the take act was refused: %v", err) + } + <-done + if !strings.Contains(said, "warrant-7") { + t.Errorf("the act does not say its warrant: %s", said) + } + for _, p := range []string{lib, filepath.Join(lib, "album"), song} { + if got := effectiveOn(t, p, radarr); got&6 != 6 { + t.Errorf("after the act the read-write identity may %s", permWords(got)) + } + } + if has := effectiveOn(t, outside, radarr); has != 0 { + t.Error("the act followed a link out of the folder") + } + rec, _ := keptRecord(lib) + if !rec.Kept || rec.How != store.KeptByWarrant || rec.Warrant != "warrant-7" || rec.Act == nil || + rec.Act.State != "done" || rec.Act.Links != 1 { + t.Fatalf("the act is not recorded as done under its warrant, with the link passed by: %+v %+v", rec, rec.Act) + } + + // Kept now: the next apply keeps it without asking, and plex dropping it has exactly plex's entries removed. + if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false, + "mesh-lidarr:61001:read-write:container")); err != nil { + t.Fatal(err) + } + beforeMask, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }() + toUser(t) + ReconcileKept(context.Background()) + for _, p := range []string{lib, song} { + a, def, _ := readACLs(t, p) + if _, still := a.perm(aclUser, plex); still { + t.Errorf("a dropped user's entry stayed") + } + if _, still := def.perm(aclUser, plex); still && def != nil { + t.Errorf("a dropped user's default entry stayed") + } + if _, kept := a.perm(aclUser, radarr); !kept { + t.Errorf("the user that stayed lost its entry") + } + } + if m, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }(); m != beforeMask { + t.Errorf("the mask moved on a drop: %s to %s", permWords(beforeMask), permWords(m)) + } +} + +// Row 8: the act enters no other filesystem — a folder whose device is not the kept folder's is passed by. +func TestAWalkEntersNoOtherFilesystem(t *testing.T) { + dir := aclFolder(t) + mount := filepath.Join(dir, "elsewhere") + if err := os.Mkdir(mount, 0o755); err != nil { + t.Fatal(err) + } + inside := filepath.Join(mount, "file") + if err := os.WriteFile(inside, nil, 0o644); err != nil { + t.Fatal(err) + } + // A stand-in for a device the test cannot mount: the folder named elsewhere reads as another filesystem. + stats := walkStats{ + at: func(dirfd int, name string, st *unix.Stat_t) error { + err := unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW) + if name == "elsewhere" { + st.Dev++ + } + return err + }, + fd: unix.Fstat, + } + toUser(t) + w, err := walkKept(dir, ActLimits, stats, nil, keepVisit(map[uint32]bool{radarr: true})) + if err != nil { + t.Fatal(err) + } + if w.counts.Mounts != 1 { + t.Errorf("the walk counted %d folders on another filesystem, and there is one", w.counts.Mounts) + } + if got := effectiveOn(t, inside, radarr); got != 0 { + t.Error("the walk entered another filesystem") + } +} + +// Row 9: a missing folder under a mounted dataset is made as its parent; with the dataset not mounted (its empty +// mount point on the root filesystem) nothing is made and the access waits. +func TestAMissingFolderIsMadeWhereItsDataLivesAndWaitsWhereItsDiskIsNot(t *testing.T) { + dir := aclFolder(t) + dataset := filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dataset, 0o750); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, []string{dataset}) + movies := filepath.Join(dataset, "media", "movies") + report, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", movies, "read-write", false, + "mesh-radarr:61001:read-write:container")) + if err != nil { + t.Fatalf("a missing folder under a mounted dataset was not made: %v", err) + } + info, err := os.Stat(movies) + if err != nil { + t.Fatalf("the folder was not made: %v", err) + } + if info.Mode().Perm()&0o707 != 0o700 { + t.Errorf("the folder was made at %o, and its parent is 0750", info.Mode().Perm()) + } + if got := effectiveOn(t, movies, radarr); got != 7 { + t.Errorf("the folder made is not kept: the identity may %s", permWords(got)) + } + if rec, _ := keptRecord(movies); rec.How != store.KeptMade || rec.Mount != dataset { + t.Errorf("not recorded as made on its dataset: %+v", rec) + } + _ = report + + // The dataset not mounted: its mount point an empty folder on the filesystem below. + MountPoints = func() []string { return []string{"/"} } + books := filepath.Join(dataset, "books") + report, _, err = applyAccesses(t, store.State{}, accessJSON("readarr.books", books, "read-write", false, + "mesh-readarr:61001:read-write:container")) + if err != nil { + t.Fatalf("a missing disk failed the apply: %v", err) + } + if o := outcomeOf(report, "readarr.books"); o.Action != "waiting" || !strings.Contains(o.Detail, DiskWords("books")) { + t.Fatalf("a folder whose disk is not there did not wait for it: %+v", o) + } + if _, err := os.Stat(books); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a folder was made where its disk is not") + } + toUser(t) + if v := AccessVerdict(context.Background(), books, false, true); v.Kind != VerdictWaitDisk { + t.Errorf("the look judges it %s, not waiting for its disk", v.Kind) + } + // And the folder made before, its dataset gone: its disk is not there, whatever is at its path now. + toUser(t) + if v := AccessVerdict(context.Background(), movies, false, true); v.Kind != VerdictWaitDisk { + t.Errorf("a kept folder whose dataset is not mounted is judged %s", v.Kind) + } +} + +// Row 10: a narrowed entry or mask is reported and put back; successive looks cover a folder whole; a look over +// its bound says how far it got. +func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "shows") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + for i := 0; i < 25; i++ { + p := filepath.Join(lib, fmt.Sprintf("episode-%02d", i)) + if err := os.WriteFile(p, nil, 0o666); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, 0o644); err != nil { // an app's chmod narrows the mask + t.Fatal(err) + } + } + // The folder itself narrowed too: put back on the next apply, said by the look until then. + if err := os.Chmod(lib, 0o755); err != nil { + t.Fatal(err) + } + toUser(t) + if short, _ := judgeTop(lib, rec.Want); len(short) == 0 { + t.Fatal("a folder whose mask an app narrowed is not said short") + } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + toUser(t) + if short, _ := judgeTop(lib, rec.Want); len(short) != 0 { + t.Fatalf("the apply did not put the folder back: %v", short) + } + + limits := WalkLimits{Entries: 10, Time: time.Minute} + looks, repaired := 0, 0 + for { + looks++ + toUser(t) + look := LookKept(lib, rec.Want, limits) + repaired += look.Repaired + if look.Failed != "" { + t.Fatal(look.Failed) + } + if !look.Sweep.Bound || look.Sweep.Cursor == nil { + break + } + if len(look.Sweep.Cursor) == 0 || look.Sweep.Read == 0 { + t.Fatalf("a look over its bound does not say how far it got: %+v", look.Sweep) + } + if looks > 10 { + t.Fatal("the sweep never covered the folder") + } + } + if looks < 3 { + t.Errorf("26 entries at 10 a look were covered in %d looks", looks) + } + if repaired != 25 { + t.Errorf("the sweep put back %d of 25 narrowed entries", repaired) + } + toUser(t) + if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { + t.Errorf("after the looks %d entries still give the user less than its right", c.Narrowed) + } + if r, _ := keptRecord(lib); r.Sweep.Covered.IsZero() { + t.Error("a sweep that covered the folder is not recorded as having done so") + } +} + +// Row 12: a root-owned folder on its placed filesystem, and an absent one under a root-owned parent on it, are +// unhealthy, nothing is made, and no ask is raised. Read from the machine's own root-owned folders, never changed. +func TestRootsDataOnItsPlacedFilesystemIsNeverKept(t *testing.T) { + keeping(t, nil, nil) + // A root-owned folder the placement guard lets an access name: below a protected root, never one. + root := "" + for _, c := range []string{"/srv/http", "/srv/ftp", "/var/empty", "/var/opt"} { + if info, err := os.Lstat(c); err == nil && info.IsDir() { + if uid, _, _ := statOwner(info); uid == 0 { + root = c + break + } + } + } + if root == "" { + t.Skip("this machine has no root-owned folder below /srv or /var to read") + } + absent := filepath.Join(root, "mesh-test-never-made") + for _, path := range []string{root, absent} { + report, _, err := applyAccesses(t, store.State{}, accessJSON("m.data", path, "read-write", false, + "mesh-m:61001:read-write:container")) + if err == nil || !strings.Contains(err.Error(), "root's data is never kept") { + t.Errorf("root's data at %s was not refused for what it is: %v", path, err) + } + if o := outcomeOf(report, "m.data"); o.Action == "waiting" { + t.Errorf("root's data waits, and is never asked about: %+v", o) + } + toUser(t) + if v := AccessVerdict(context.Background(), path, false, true); v.Kind != VerdictRoot { + t.Errorf("the look judges root's data at %s %s", path, v.Kind) + } + } + if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a folder was made under root's data") + } + if _, has := keptRecord(root); has { + t.Error("root's data was recorded as a kept folder") + } +} + +// The codec reads back what it writes, in the kernel's order. +func TestAnACLReadsBackAsWritten(t *testing.T) { + a := keptACL(minimalACL(0o640), map[uint32]uint16{radarr: 6, plex: 4}) + b, err := decodeACL(a.encode()) + if err != nil { + t.Fatal(err) + } + if !sameACL(a, b) || b[0].Tag != aclUserObj || b[len(b)-1].Tag != aclOther { + t.Fatalf("read back %+v", b) + } + if err := b.check(); err != nil { + t.Fatal(err) + } + if _, err := decodeACL([]byte{1, 0, 0, 0}); err == nil { + t.Error("a version 1 attribute was read") + } + if m, _ := b.perm(aclMask, 0); m != 6 { + t.Errorf("mask %s, and the entries need rw- and the group had r--", permWords(m)) + } +} + +func forgetMountHistory() { + mountHistory.Lock() + mountHistory.seen = map[string]bool{} + mountHistory.Unlock() +} + +// Review H1: a walk is judged as an apply is, first. With the dataset not mounted its mount point is what is at the +// path, and neither an add nor a drop walk touches it; the record stays for the walk that can reach the data. +func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) { + dir := aclFolder(t) + dataset := filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib := filepath.Join(dataset, "movies") + both := []string{"mesh-radarr:61001:read-write:container", "mesh-plex:61002:read:container"} + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both...)); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); rec.Mount != dataset || !sameUsers(rec.Users, rec.Want) { + t.Fatalf("not kept on its dataset: %+v", rec) + } + // plex drops it, and the dataset is unmounted before the walk. + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both[0])); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + toUser(t) + ReconcileKept(context.Background()) + rec, had := keptRecord(lib) + if !had || len(rec.Users) != 2 { + t.Fatalf("a drop that could not reach the data forgot whose entries are on it: %+v", rec) + } + if rec.Act == nil || rec.Act.State != "waiting" { + t.Errorf("the walk not run is not said: %+v", rec.Act) + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); !still { + t.Fatal("a walk ran where the disk is not there") + } + // The disk back: the drop runs. + MountPoints = func() []string { return []string{"/", dataset} } + toUser(t) + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); still { + t.Fatal("the drop did not run once the disk was there") + } +} + +// Review H2: a folder kept while it was empty on the filesystem below a mount nobody named is judged afresh when +// the disk mounts over it: holding data, it waits for the operator's word, and nothing is added to it. +func TestADiskMountedOverAFolderKeptEmptyIsNotKeptWithoutAWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); !rec.Kept || rec.How != store.KeptEmpty || rec.Mount != "/" { + t.Fatalf("not kept as empty on /: %+v", rec) + } + // The pool is imported: a library is at the path now, on its own filesystem. + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + // The new disk's data carries no entries of the mesh's: the test's stand-in made it under the folder kept + // below, so what it inherited there is taken off. + if err := unix.Removexattr(film, aclAccessXattr); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/", pool} } + toUser(t) + if v := AccessVerdict(context.Background(), lib, false, true); v.Kind != VerdictWaitKeep { + t.Fatalf("the look judges it %s, and the data on the new disk was never taken", v.Kind) + } + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder whose disk mounted over it was kept without a warrant: %+v", o) + } + if rec, _ := keptRecord(lib); rec.Kept || rec.Mount != pool { + t.Errorf("the record still says kept, or not on the new disk: %+v", rec) + } + if got := effectiveOn(t, film, radarr); got != 0 { + t.Error("an entry was added to the data on the new disk") + } +} + +// Review M3: widening a mask widens nobody else — a file made at 0644 under a default ACL carries the owning group's +// rwx under a mask of r--, and after the mesh widens the mask the owning group still reads only. +func TestWideningTheMaskGivesTheOwningGroupNothingMore(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "shows") + if err := os.Mkdir(lib, 0o775); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lib, 0o775); err != nil { + t.Fatal(err) + } + a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + ep := filepath.Join(lib, "episode") + f, err := os.OpenFile(ep, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.Close() + groupRight := func() uint16 { + acl, _, _ := readACLs(t, ep) + g, _ := acl.perm(aclGroupObj, 0) + m, _ := acl.perm(aclMask, 0) + return g & m + } + before := groupRight() + rec, _ := keptRecord(lib) + toUser(t) + LookKept(lib, rec.Want, SweepLimits) + if got := effectiveOn(t, ep, radarr); got&6 != 6 { + t.Fatalf("the sweep did not widen the mask: %s", permWords(got)) + } + if after := groupRight(); after != before { + t.Fatalf("the owning group may %s after the mask widened, and %s before", permWords(after), permWords(before)) + } +} + +// watchedFolder is a kept folder with a watch running on it, its users, and the end of the watch's run. +func watchedFolder(t *testing.T) (string, []store.KeptUser, *Watcher, context.CancelFunc, <-chan struct{}) { + t.Helper() + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "music") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false, + "mesh-lidarr:61001:read-write:container")); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + w, err := NewWatcher() + if err != nil { + t.Skipf("no inotify here: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + // The watch ends before the test's record of kept folders is put back. + t.Cleanup(func() { cancel(); <-ended }) + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + return lib, rec.Want, w, cancel, ended +} + +// Review L4: a watch whose folder goes is let go (IN_IGNORED). +func TestTheWatchOfAFolderRemovedIsLetGo(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + album := filepath.Join(lib, "album") + if err := os.Mkdir(album, 0o755); err != nil { + t.Fatal(err) + } + waitFor(t, "the folder made is watched", func() bool { return w.watched() == 2 }) + if err := os.Remove(album); err != nil { + t.Fatal(err) + } + waitFor(t, "the watch of a folder removed is let go", func() bool { return w.watched() == 1 }) +} + +// Review L4: notice lost re-arms every watch, and the record says the watch is whole again. +func TestNoticeLostReArmsTheWatch(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + w.lost() + if w.watched() != 1 { + t.Fatalf("notice lost did not re-arm the watch: %d watched", w.watched()) + } + if r, _ := keptRecord(lib); r.OutOfWatches { + t.Error("a re-armed watch is still said out of watches") + } + late := filepath.Join(lib, "late") + f, _ := os.OpenFile(late, os.O_CREATE|os.O_WRONLY, 0o644) + f.Close() + waitFor(t, "the re-armed watch repairs", func() bool { return effectiveOn(t, late, radarr)&6 == 6 }) +} + +// Review L4: users given while notices are heard are read under the lock (run with -race). +func TestUsersGivenWhileNoticesAreHeardAreReadUnderTheLock(t *testing.T) { + lib, users, w, _, _ := watchedFolder(t) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + w.Set(map[string][]store.KeptUser{lib: users}) + } + } + }() + for i := 0; i < 50; i++ { + _ = os.WriteFile(filepath.Join(lib, fmt.Sprintf("f%d", i)), nil, 0o644) + } + waitFor(t, "the last file is repaired", func() bool { + return effectiveOn(t, filepath.Join(lib, "f49"), radarr)&6 == 6 + }) + close(stop) + <-done +} + +// Review L4: the watch ends with its context, however quiet the machine. +func TestTheWatchEndsWithItsContext(t *testing.T) { + _, _, _, cancel, ended := watchedFolder(t) + cancel() + select { + case <-ended: + case <-time.After(2 * time.Second): + t.Fatal("the watch did not end with its context") + } +} + +// Review L6: a broken folder's words carry no path. +func TestABrokenFoldersWordsCarryNoPath(t *testing.T) { + dir := userDir(t) + keeping(t, nil, nil) + file := filepath.Join(dir, "notafolder") + if err := os.WriteFile(file, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, path := range []string{file, filepath.Join(file, "below")} { + toUser(t) + v := AccessVerdict(context.Background(), path, false, true) + if v.Kind != VerdictBroken || strings.Contains(v.Why, "/") { + t.Errorf("%s: %+v", path, v) + } + } +} + +// Review H2: a systemd .mount unit says where a disk is meant to be, as fstab does; and a mount point seen mounted +// once is remembered, so a folder below it is a missing disk on first sight when it is gone. +func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) { + units := t.TempDir() + unit := "[Unit]\nDescription=pool\n\n[Mount]\nWhat=/dev/disk/by-label/pool\nWhere=/srv/pool\nType=ext4\n" + if err := os.WriteFile(filepath.Join(units, "srv-pool.mount"), []byte(unit), 0o644); err != nil { + t.Fatal(err) + } + if got := mountUnitTargets([]string{units}); len(got) != 1 || got[0] != "/srv/pool" { + t.Fatalf("read %v from a .mount unit", got) + } + + dir := t.TempDir() + disk := filepath.Join(dir, "disk") + if err := os.Mkdir(disk, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{disk}, nil) + _ = placedOn(context.Background(), disk, disk, "", nil) // seen mounted once + MountPoints = func() []string { return []string{"/"} } + if pl := placedOn(context.Background(), filepath.Join(disk, "films"), disk, "", nil); !pl.Away { + t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk") + } +} + +// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset. +func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) { + t.Helper() + dir := aclFolder(t) + dataset = filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib = filepath.Join(dataset, "movies") + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + users...)); err != nil { + t.Fatal(err) + } + return dataset, lib +} + +// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount +// point is at the path: its writes would land on the filesystem below (§9). +func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) { + _, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + MountPoints = func() []string { return []string{"/"} } + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + if len(args) > 0 && args[0] == "container" { + return "", errors.New("no such container") + } + return "", nil + } + d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+ + `,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`) + toUser(t) + report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) + if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" { + t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o) + } + for _, r := range ran { + if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") { + t.Fatalf("the container was started: %s", r) + } + } + if !WaitingOnAccess(d)["radarr.app"] { + t.Error("liveness is not told the container waits on its access") + } +} + +// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until +// the drop walk can run; then it is forgotten. +func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { + dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}` + if _, _, err := applyAccesses(t, store.State{}, other); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + toUser(t) + ReconcileKept(context.Background()) + left := Leftovers() + if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") || + strings.Contains(left[0].Why, dataset) { + t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left) + } + MountPoints = func() []string { return []string{"/", dataset} } + toUser(t) + ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("said after the drop ran: %+v", left) + } + if _, had := keptRecord(lib); had { + t.Error("a folder nobody uses was not forgotten once its entries were out") + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { + t.Error("the entries were not taken out") + } +} + +// Review L3: a disk mounted over a folder taken below it on a warrant leaves the mesh's entries on that data +// recorded and said; once that filesystem holds the folder again, they are taken out. +func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + toUser(t) + done, _, err := TakeFolder(context.Background(), declare(t, a), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") { + t.Fatalf("entries left on the data taken below are not said: %+v", left) + } + // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. + MountPoints = func() []string { return []string{"/"} } + toUser(t) + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { + a, _, _ := readACLs(t, filepath.Join(lib, "old")) + return a.perm(aclUser, radarr) + }(); still { + t.Fatal("the entries left on the filesystem that came back were not taken out") + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("still said after they were taken out: %+v", left) + } +} + +// Review (third) M: a folder kept only because it was empty leaves nothing said when a disk mounts over it — its +// entries sit on an empty folder hidden under the mount and grant nothing on data. +func TestAFolderKeptEmptyLeavesNoLeftoverWhenADiskMountsOverIt(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/", pool} } + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a folder kept while empty is said as leftover entries for ever: %+v", left) + } + if rec, _ := keptRecord(lib); len(rec.Left) != 0 { + t.Errorf("entries on the hidden empty folder are recorded: %+v", rec.Left) + } + if o := outcomeOf(report, "radarr.media"); !strings.Contains(o.Detail, "forgotten") { + t.Errorf("the forgetting is not said: %+v", o) + } +} + +// Review (third) low: a record written before it kept the accesses' ids is said too, under a name with no path. +func TestALeftoverWithNoAccessIdsIsSaidUnderAPathFreeName(t *testing.T) { + keeping(t, nil, nil) + path := "/srv/media/old" + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Kept, f.Users = true, []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}} + f.Act = &store.KeptAct{State: "waiting"} + return true + }) + left := Leftovers() + if len(left) != 1 || left[0].Module != "radarr" || strings.Contains(left[0].Why, path) || + strings.Contains(left[0].Access, "/") { + t.Fatalf("a record without ids is not said, path-free: %+v", left) + } +} + +// placedPool is a folder of a library on a pool the test says is mounted, with the pool's device name and filesystem +// id as the test sets them, taken under a warrant. +func placedPool(t *testing.T, source, fsid *string) (pool, lib, taken string) { + t.Helper() + dir := aclFolder(t) + pool = filepath.Join(dir, "pool") + lib = filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{pool}, nil) + wasSource, wasFsid := MountSource, MountFsid + MountSource = func(point string) (string, string) { + if point == pool { + return *source, "ext4" + } + return "", "" + } + MountFsid = func(point string) string { + if point == pool { + return *fsid + } + return "" + } + t.Cleanup(func() { MountSource, MountFsid = wasSource, wasFsid }) + notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { + t.Fatal(err) + } + toUser(t) + done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + taken = accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { + t.Fatal("the folder taken waits on its own filesystem") + } + return pool, lib, taken +} + +// Review (third) low: another filesystem mounted at the same point is not the one the warrant was for. +func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + _, lib, taken := placedPool(t, &source, &fsid) + fsid = "1111222233334444" // another filesystem mounted at the same point + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("another filesystem at the same mount point was kept on the old warrant: %+v", o) + } + toUser(t) + if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { + t.Fatalf("the look judges it %s", v.Kind) + } +} + +// Review (fourth): the same filesystem whose device the kernel renamed across a reboot (/dev/sdb1 to /dev/sdc1, one +// filesystem id) is the same filesystem: still taken, still kept, and none of its entries walked away. +func TestADeviceRenamedIsTheSameFilesystem(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + _, lib, taken := placedPool(t, &source, &fsid) + source = "/dev/sdc1" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("a device renamed reads as another filesystem: %+v", o) + } + toUser(t) + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("the library's entries were walked away: the identity may %s", permWords(got)) + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a device renamed leaves entries said as left: %+v", left) + } +} + +// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on; +// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked. +func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { + t.Fatal(err) + } + toUser(t) + done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + toUser(t) + before := AccessVerdict(context.Background(), lib, true, true) + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o) + } + // And on the next apply too, its record now on the new filesystem. + report, _, _ = applyAccesses(t, store.State{}, taken) + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("the second apply kept it on the old warrant: %+v", o) + } + toUser(t) + v := AccessVerdict(context.Background(), lib, true, true) + if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") { + t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before) + } + toUser(t) + ReconcileKept(context.Background()) + if got := effectiveOn(t, film, radarr); got != 0 { + t.Fatal("the data on the new filesystem was walked") + } +} + +// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile. +func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) { + was := statBound + statBound = 20 * time.Millisecond + defer func() { statBound = was }() + release := make(chan struct{}) + defer close(release) + var started int32 + var mu sync.Mutex + hang := func(string) (os.FileInfo, error) { + mu.Lock() + started++ + mu.Unlock() + <-release + return nil, nil + } + for i := 0; i < 5; i++ { + if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) { + t.Fatalf("a hung read answered: %v", err) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one path were started while the first hung", started) + } +} + +// Review (fourth): entries left on the filesystem the folder is on now are never walked away — merged back into +// the folder's users when it is kept there, held while it is not. +func TestEntriesLeftOnTheFilesystemTheFolderIsOnAreNeverWalkedAway(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + pool, lib, _ := placedPool(t, &source, &fsid) + rec, _ := keptRecord(lib) + users := rec.Users + for _, kept := range []bool{false, true} { + _ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool { + f.Kept, f.Users = kept, nil + f.Left = []store.KeptLeft{{Mount: pool, Placed: f.Placed, Users: users}} + return true + }) + toUser(t) + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("kept=%v: entries on the folder's own filesystem were walked away", kept) + } + r, _ := keptRecord(lib) + if kept && (len(r.Left) != 0 || len(r.Users) != 1) { + t.Errorf("kept: not merged back into the folder's users: %+v", r) + } + if !kept && len(r.Left) != 1 { + t.Errorf("not kept: the entries' record was dropped: %+v", r) + } + } +} + +// Review (fourth): a TakenOn naming a mount point (an unmerged build's record) is not taken on whatever is mounted +// there now. +func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + _ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.TakenOn = "/", "/" + return true + }) + toUser(t) + if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { + t.Fatalf("an old-form record is read as taken: %s", v.Kind) + } +} + +// poolNamed is a library on a pool the test says is mounted, whose UUID, filesystem id and mountinfo source are as +// the test sets them; the access declared, not taken. +func poolNamed(t *testing.T, uuid, fsid, source *string) (pool, lib, access string) { + t.Helper() + dir := aclFolder(t) + pool = filepath.Join(dir, "pool") + lib = filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{pool}, nil) + wasUUID, wasFsid, wasSource := MountUUID, MountFsid, MountSource + MountUUID = func(p string) string { return pick(p == pool, *uuid) } + MountFsid = func(p string) string { return pick(p == pool, *fsid) } + MountSource = func(p string) (string, string) { return pick(p == pool, *source), pick(p == pool, "nfs4") } + t.Cleanup(func() { MountUUID, MountFsid, MountSource = wasUUID, wasFsid, wasSource }) + return pool, lib, accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") +} + +func pick(yes bool, v string) string { + if yes { + return v + } + return "x" +} + +// takeIt takes the folder of access under a warrant, and says it is kept. +func takeIt(t *testing.T, lib, access string) { + t.Helper() + if _, _, err := applyAccesses(t, store.State{}, access); err != nil { + t.Fatal(err) + } + toUser(t) + done, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done +} + +// Review (fifth) M: an NFS export (f_fsid 0, no UUID) is named by its source; another export at the same mount +// point is another filesystem, not taken. +func TestAnNFSExportIsNamedByItsSource(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "nas:/export/media" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { + t.Fatal("the export taken waits on its own filesystem") + } + source = "other-nas:/export/media" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("another export at the same mount point was kept on the old warrant: %+v", o) + } +} + +// Review (fifth) M: a filesystem that cannot be named is never kept, nor taken: it fails closed, and waits. +func TestAFilesystemThatCannotBeNamedIsNeverKept(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + if err := os.RemoveAll(filepath.Join(lib, "old")); err != nil { // empty: it would be kept without asking + t.Fatal(err) + } + report, _, err := applyAccesses(t, store.State{}, access) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder on a filesystem that cannot be named was kept: %+v", o) + } + if got := effectiveOn(t, lib, radarr); got != 0 { + t.Fatal("an entry was added on a filesystem that cannot be named") + } + toUser(t) + if _, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w", "op", noServices); err == nil { + t.Fatal("the take act ran on a filesystem that cannot be named") + } +} + +// Review (fifth): xfs's f_fsid follows the device number; its UUID names it first, so a renumbered device is the same +// filesystem. +func TestAFilesystemsUUIDNamesItBeforeItsFsid(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + fsid, source = "0000082100000000", "/dev/sdc1" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("a renumbered device with one UUID reads as another filesystem: %+v", o) + } + toUser(t) + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("the library's entries were walked away: %s", permWords(got)) + } +} + +// Review (fifth): entries left on a filesystem that never comes back are forgotten after LeftFor, said once, and no +// longer said unhealthy. +func TestEntriesLeftOnAFilesystemThatNeverReturnsAreForgotten(t *testing.T) { + keeping(t, nil, nil) + path := "/srv/media/gone" + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.IDs = []string{"radarr.media"} + f.Left = []store.KeptLeft{{Mount: "/srv/media", Placed: "fs-000000000000", Fs: "uuid:gone", + Since: time.Now().Add(-LeftFor - time.Hour), + Users: []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}}}} + return true + }) + if len(Leftovers()) != 1 { + t.Fatal("the leftover is not said before it expires") + } + toUser(t) + said := ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a leftover on a filesystem gone for longer than %s is still said: %+v", LeftFor, left) + } + if !strings.Contains(strings.Join(said, "\n"), "forgotten") { + t.Errorf("the forgetting is not said: %v", said) + } +} + +// Review (fifth) low: naming a filesystem whose read hangs holds one read per mount, whoever asks meanwhile. +func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) { + was, wasBound := MountUUID, statBound + statBound = 20 * time.Millisecond + release := make(chan struct{}) + var mu sync.Mutex + started := 0 + MountUUID = func(string) string { + mu.Lock() + started++ + mu.Unlock() + <-release + return "" + } + defer func() { close(release); flightsDone(t); MountUUID, statBound = was, wasBound }() + for i := 0; i < 5; i++ { + if id := PlacedID("/a/dead/mount"); id != "" { + t.Fatalf("a filesystem that does not answer is named %s", id) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one mount were started while the first hung", started) + } +} + +// Review (sixth): a name read that does not answer is unknown, never a move — a folder kept while empty and filled +// since is not asked about again, a warranted one is not walked again, and nothing of either record is reset. +func TestAFilesystemNotNamedThisTimeIsNeverAMove(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + // The name's reads hang past their bound while hanging is set; installed once, before anything reads it. + var hanging atomic.Bool + release := make(chan struct{}) + named := MountUUID + MountUUID = func(p string) string { + if hanging.Load() { + <-release + } + return named(p) + } + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + before, _ := keptRecord(lib) + + wasBound := statBound + statBound = 20 * time.Millisecond + hanging.Store(true) + report, _, err := applyAccesses(t, store.State{}, taken) + hanging.Store(false) + close(release) + flightsDone(t) + statBound = wasBound + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" || !strings.Contains(o.Detail, "could not be named") { + t.Fatalf("a folder whose filesystem was not named this time is not said so: %+v", o) + } + after, _ := keptRecord(lib) + if !after.Kept || after.Placed != before.Placed || after.TakenOn != before.TakenOn || len(after.Left) != 0 || + !sameUsers(after.Users, before.Users) { + t.Fatalf("the record was reset by a name not read:\nbefore %+v\nafter %+v", before, after) + } + + // Named again: kept as it was, with nothing asked and nothing walked. + waitFor(t, "the hung read returned", func() bool { return PlacedID(filepath.Dir(lib)) == before.Placed }) + report, _, err = applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("named again, the folder asks again: %+v", o) + } + if r, _ := keptRecord(lib); r.Act == nil || r.Act.Act != "take" { + t.Errorf("the folder was walked again: %+v", r.Act) + } +} + +// flightsDone waits until no read of a hung test is in flight: it goes on reading the stand-ins after its release. +func flightsDone(t *testing.T) { + t.Helper() + waitFor(t, "the hung reads returned", func() bool { + inFlight.Lock() + defer inFlight.Unlock() + return len(inFlight.reads) == 0 + }) +} + +// userDirs are the folders the running test made, which toUser gives an unprivileged owner when it runs as root. +var userDirs []string + +// userDir is a t.TempDir that toUser gives an unprivileged owner when the test runs as root (the build seat runs +// merge-check as root): root's data is never kept (ADR 0325 §12), and a test's own folders stand for an operator's. +func userDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + userDirs = append(userDirs, dir) + t.Cleanup(func() { userDirs = nil }) + return dir +} + +// unprivileged is the owner toUser gives a test's folders: nobody. +const unprivileged = 65534 + +// toUser gives everything under the running test's own folders to an unprivileged owner when the test runs as +// root, so they are judged as an operator's data is; as anyone else it changes nothing. +func toUser(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + return + } + for _, dir := range userDirs { + err := filepath.WalkDir(dir, func(p string, _ os.DirEntry, err error) error { + if err != nil { + return err + } + return os.Lchown(p, unprivileged, unprivileged) + }) + if err != nil { + t.Fatalf("giving the test's folders to an unprivileged owner: %v", err) + } + } +} diff --git a/internal/apply/mesh_names_test.go b/internal/apply/mesh_names_test.go index 54ff8cd..794911f 100644 --- a/internal/apply/mesh_names_test.go +++ b/internal/apply/mesh_names_test.go @@ -23,7 +23,7 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) { Name: was.Name, Image: was.Image, Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"}, } - if containerSpecReading(was, nil, nil, nil) == containerSpecReading(moved, nil, nil, nil) { + if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) { t.Fatal("a container whose mesh names moved compares equal, so it is never recreated") } @@ -33,14 +33,14 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) { Name: moved.Name, Image: moved.Image, Hosts: []string{moved.Hosts[1], moved.Hosts[0]}, } - if containerSpecReading(moved, nil, nil, nil) != containerSpecReading(reordered, nil, nil, nil) { + if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) { t.Fatal("the same names in another order read as a different container") } // A container the mesh gives no names is unaffected, so nothing is recreated for a field it // does not set. plain := &declaration.Container{Name: "plex", Image: was.Image} - if containerSpecReading(plain, nil, nil, nil) == containerSpecReading(was, nil, nil, nil) { + if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) { return // different for other reasons, which is fine } } diff --git a/internal/apply/oldshape_test.go b/internal/apply/oldshape_test.go new file mode 100644 index 0000000..a4eb521 --- /dev/null +++ b/internal/apply/oldshape_test.go @@ -0,0 +1,136 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The engine delivers before the controller half (novox/hq ADR 0325): a controller that does not speak +// link.IdentityContract sends no identities, no users and no taken, and every access of its declarations is applied +// exactly as before this engine — mounted when present, refused when absent (ADR 0051 §7), nothing kept, nothing +// made, no container held, nothing judged. These are origin/main's outcomes, word for word. +func TestAnOlderControllersAccessesAreAppliedExactlyAsBefore(t *testing.T) { + keeping(t, nil, nil) + dir := userDir(t) + unwritable := filepath.Join(dir, "library") // its group may not write it + empty := filepath.Join(dir, "spool") + absent := filepath.Join(dir, "media", "movies") + for _, d := range []string{unwritable, empty} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(d, 0o755); err != nil { + t.Fatal(err) + } + } + // A root-owned empty folder, read only. + rootEmpty := "" + for _, c := range []string{"/var/empty", "/srv/http", "/srv/ftp"} { + if entries, err := os.ReadDir(c); err == nil && len(entries) == 0 { + if uid, _, _ := statOwner(mustLstat(t, c)); uid == 0 { + rootEmpty = c + break + } + } + } + resources := []string{ + `{"id":"radarr.library","type":"access","path":"` + unwritable + `","mode":"read-write"}`, + `{"id":"sab.spool","type":"access","path":"` + empty + `","mode":"read-write"}`, + `{"id":"plex.movies","type":"access","path":"` + absent + `"}`, + `{"id":"plex.app","type":"container","name":"plex","image":"` + pinned + `","volumes":["` + absent + `:/movies"]}`, + } + if rootEmpty != "" { + resources = append(resources, `{"id":"web.root","type":"access","path":"`+rootEmpty+`","mode":"read-write"}`) + } + var started bool + run := func(_ context.Context, name string, args ...string) (string, error) { + if len(args) > 0 && args[0] == "run" { + started = true + } + if len(args) > 0 && args[0] == "container" { + if started { + return "true\t" + containerSpec(declare(t, resources[3]).Resources[0].(*declaration.Container), inputs{}), nil + } + return "", errors.New("no such container") + } + return "", nil + } + toUser(t) + report, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")), store.State{}, + store.OriginDeclared, run, nil, nil) + + if err == nil || !strings.Contains(err.Error(), absent+" is not there, and the mesh does not own it — the operator "+ + "must provide it") { + t.Fatalf("an absent access is not refused as before: %v", err) + } + for id, mode := range map[string]string{"radarr.library": "read-write", "sab.spool": "read-write"} { + o := outcomeOf(report, id) + if o.Action != "unchanged" || o.Detail != "operator-owned; present, "+mode+", nothing managed" { + t.Errorf("%s: %+v, and before it was unchanged, %q", id, o, "operator-owned; present, "+mode+", nothing managed") + } + } + if rootEmpty != "" { + if o := outcomeOf(report, "web.root"); o.Action != "unchanged" { + t.Errorf("a root-owned folder was judged: %+v", o) + } + } + if !started { + t.Error("a container mounting an absent access was held, and before it was created") + } + if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) { + t.Error("an absent access was made") + } + for _, d := range []string{unwritable, empty} { + if n, _ := unix.Getxattr(d, aclAccessXattr, nil); n > 0 { + t.Errorf("an ACL was set on %s", d) + } + if info := mustLstat(t, d); info.Mode().Perm() != 0o755 { + t.Errorf("%s changed mode: %o", d, info.Mode().Perm()) + } + } + if all, _ := KeptFolders.All(); len(all) != 0 { + t.Errorf("folders were recorded as kept: %v", all) + } + if len(Leftovers()) != 0 || len(WaitingOnAccess(declare(t, strings.Join(resources, ",")))) != 0 { + t.Error("something was said or held for an older controller's access") + } +} + +func mustLstat(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + return info +} + +// Review (low): a folder no access names, whose filesystem could not be named this look, is not said as its disk +// being away. +func TestALeftoverSaysWhyItsWalkCouldNotRun(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(filepath.Dir(lib)), "x") + `"}` + if _, _, err := applyAccesses(t, store.State{}, other); err != nil { + t.Fatal(err) + } + uuid, fsid, source = "", "0000000000000000", "" // nothing names it now + toUser(t) + ReconcileKept(context.Background()) + left := Leftovers() + if len(left) != 1 || strings.Contains(left[0].Why, "disk") || !strings.Contains(left[0].Why, "could not be named") { + t.Fatalf("the leftover does not say why its walk could not run: %+v", left) + } +} diff --git a/internal/apply/placed.go b/internal/apply/placed.go new file mode 100644 index 0000000..8008bda --- /dev/null +++ b/internal/apply/placed.go @@ -0,0 +1,261 @@ +package apply + +import ( + "bufio" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// Whether a shared folder's disk is there (novox/hq ADR 0325 §9). +// +// **The rule the engine keeps.** A folder is placed on the filesystem it is first seen on: the mount point that +// holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the +// kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged +// against that record: held by another mount point than the one recorded while the recorded one is not mounted is +// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is +// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made +// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's +// word as any folder holding data does (ADR 0325 §8). +// +// **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be: +// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names +// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every +// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point +// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what +// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list` +// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen +// empty below it is kept as empty, and the move above takes it back the moment the disk is there. + +// MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test +// can stand in for mounts it cannot make. +var MountPoints = func() []string { + return readMountPoints() +} + +// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history"). +var mountHistory = struct { + sync.Mutex + seen map[string]bool +}{seen: map[string]bool{}} + +// sawMounted adds points to the history, and is the history: every mount point seen, never "/". +func sawMounted(points []string) []string { + mountHistory.Lock() + defer mountHistory.Unlock() + for _, p := range points { + if p != "/" { + mountHistory.seen[p] = true + } + } + out := make([]string, 0, len(mountHistory.seen)) + for p := range mountHistory.seen { + out = append(out, p) + } + return out +} + +func readMountPoints() []string { + raw, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return []string{"/"} + } + return mountinfoPoints(string(raw)) +} + +// MountTargets is every place a disk is meant to be mounted: /etc/fstab's targets and ZFS datasets' mount points, +// read at most once a minute — every look judges every access by them. A variable so a test can stand in. +var MountTargets = func(ctx context.Context) []string { + targets.Lock() + defer targets.Unlock() + if targets.read.IsZero() || time.Since(targets.read) > time.Minute { + targets.list, targets.read = readMountTargets(ctx), time.Now() + } + return append([]string(nil), targets.list...) +} + +var targets struct { + sync.Mutex + list []string + read time.Time +} + +func readMountTargets(ctx context.Context) []string { + var out []string + if raw, err := os.ReadFile("/etc/fstab"); err == nil { + out = append(out, fstabTargets(string(raw))...) + } + out = append(out, mountUnitTargets(systemdUnitDirs)...) + if _, err := exec.LookPath("zfs"); err == nil { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if raw, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "mountpoint").Output(); err == nil { + for _, line := range strings.Split(string(raw), "\n") { + if p := strings.TrimSpace(line); strings.HasPrefix(p, "/") { + out = append(out, filepath.Clean(p)) + } + } + } + } + return out +} + +// systemdUnitDirs are where systemd keeps the units a machine mounts by. +var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator", + "/usr/lib/systemd/system", "/lib/systemd/system"} + +// mountUnitTargets reads the `Where=` of every .mount unit in dirs. +func mountUnitTargets(dirs []string) []string { + var out []string + for _, dir := range dirs { + names, err := filepath.Glob(filepath.Join(dir, "*.mount")) + if err != nil { + continue + } + for _, name := range names { + raw, err := os.ReadFile(name) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") { + out = append(out, filepath.Clean(strings.TrimSpace(v))) + } + } + } + } + return out +} + +// mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes +// undone. +func mountinfoPoints(raw string) []string { + var out []string + for _, line := range strings.Split(raw, "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + out = append(out, filepath.Clean(unescapeMount(fields[4]))) + } + return out +} + +// mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its +// name. Empty when the machine says nothing of it. +func mountSourceOf(point string) (source, fstype string) { return MountSource(point) } + +// MountSource is mountSourceOf as the machine answers it. A variable so a test can stand in for another filesystem +// mounted at the same point. +var MountSource = func(point string) (source, fstype string) { + raw, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return "", "" + } + for _, line := range strings.Split(string(raw), "\n") { + before, after, ok := strings.Cut(line, " - ") + fields, rest := strings.Fields(before), strings.Fields(after) + if !ok || len(fields) < 5 || len(rest) < 2 { + continue + } + if filepath.Clean(unescapeMount(fields[4])) == point { + source, fstype = unescapeMount(rest[1]), rest[0] + } + } + return source, fstype +} + +// fstabTargets reads the second field of each fstab line that mounts somewhere: not swap, not a comment. +func fstabTargets(raw string) []string { + var out []string + sc := bufio.NewScanner(strings.NewReader(raw)) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || !strings.HasPrefix(fields[1], "/") { + continue + } + out = append(out, filepath.Clean(unescapeMount(fields[1]))) + } + return out +} + +// unescapeMount undoes the kernel's and fstab's octal escapes: \040 is a space. +func unescapeMount(s string) string { + if !strings.Contains(s, `\`) { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+4 <= len(s) { + if v, ok := octal3(s[i+1 : i+4]); ok { + b.WriteByte(v) + i += 3 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +func octal3(s string) (byte, bool) { + if len(s) != 3 { + return 0, false + } + v := 0 + for _, c := range s { + if c < '0' || c > '7' { + return 0, false + } + v = v*8 + int(c-'0') + } + return byte(v), v < 256 +} + +// mountHolding is the longest of points that is path or above it: the mount point that holds it. +func mountHolding(path string, points []string) string { + best := "/" + for _, p := range points { + if atOrBelow(path, p) && len(p) > len(best) { + best = p + } + } + return best +} + +// placement is the judgement of where a folder's data lives. +type placement struct { + // Mount is the mount point that holds what is there: the folder, or its nearest existing parent. + Mount string + // Away is that the disk the folder was placed on is not there. + Away bool + // Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what + // was kept. + Moved bool +} + +// placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent), +// against what was recorded and where disks are meant to be. +func placedOn(ctx context.Context, path, existing, recorded string, others []string) placement { + points := MountPoints() + mounted := map[string]bool{} + for _, p := range points { + mounted[p] = true + } + have := mountHolding(existing, points) + seen := sawMounted(points) + if recorded != "" { + return placement{Mount: have, Away: have != recorded && !mounted[recorded], + Moved: have != recorded && mounted[recorded]} + } + for _, t := range append(append(MountTargets(ctx), others...), seen...) { + if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] { + return placement{Mount: have, Away: true} + } + } + return placement{Mount: have} +} diff --git a/internal/apply/plan.go b/internal/apply/plan.go index 9967a2f..6385eb7 100644 --- a/internal/apply/plan.go +++ b/internal/apply/plan.go @@ -112,7 +112,7 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { step.Verb, step.Why = "restore", "no longer declared; the original the mesh wrote over goes back "+ "from "+orphan.Kept+", unless the file was changed since the mesh last wrote it" case orphan.Type == string(declaration.TypeAccess) && orphan.User != "" && len(orphan.Groups) > 0: - // In removeAccess's words (novox/hq ADR 0312 §5). + // In takeBackGranted's words (novox/hq ADR 0325 §5: no account is in a group for data). step.Verb, step.Why = "restore", "no longer declared; the path is left as it is, and "+orphan.User+ " leaves "+strings.Join(orphan.Groups, ", ")+", which the mesh put it in, unless another declared "+ "resource still asks for it" diff --git a/internal/apply/reads_test.go b/internal/apply/reads_test.go index deaf6f3..be2d8fe 100644 --- a/internal/apply/reads_test.go +++ b/internal/apply/reads_test.go @@ -212,7 +212,7 @@ func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *te t.Fatal(err) } d := declare("5432") - legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil, nil) + legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil) known := store.State{} known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: digestOf("DATABASE_PORT=5432\n")}) diff --git a/internal/apply/roots.go b/internal/apply/roots.go new file mode 100644 index 0000000..8dfccb6 --- /dev/null +++ b/internal/apply/roots.go @@ -0,0 +1,147 @@ +package apply + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/user" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/accounts" +) + +// Root's numbers, which no module identity is and whose data is never kept (novox/hq ADR 0325 §11, §12). +// +// A module identity whose uid or gid is 0, or whose gid is a group whose members act as root (accounts.RootGroups, +// the account judge's own closed list), would be a way to root through a container or a shared folder: refused, +// naming why. A shared folder owned by root or such a group is the machine's own data, never kept. + +// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine +// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine +// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number +// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and +// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can +// stand in. +var RootGroupIDs = func() map[int]string { + rootIDs.Lock() + defer rootIDs.Unlock() + if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute { + rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now() + } + return rootIDs.ids +} + +var rootIDs struct { + sync.Mutex + ids map[int]string + at time.Time +} + +func readRootGroupIDs() map[int]string { + out := map[int]string{} + if raw, err := os.ReadFile("/etc/group"); err == nil { + for name, gid := range groupEntries(string(raw)) { + if slices.Contains(accounts.RootGroups, name) { + out[gid] = name + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for _, name := range accounts.RootGroups { + raw, err := exec.CommandContext(ctx, "getent", "group", name).Output() + if err != nil { + continue + } + for found, gid := range groupEntries(string(raw)) { + if found == name { + out[gid] = name + } + } + } + return out +} + +// groupEntries reads group database lines, name:password:gid:members, as name to number. +func groupEntries(raw string) map[string]int { + out := map[string]int{} + for _, line := range strings.Split(raw, "\n") { + fields := strings.Split(strings.TrimSpace(line), ":") + if len(fields) < 3 || strings.HasPrefix(fields[0], "#") { + continue + } + if gid, err := strconv.Atoi(fields[2]); err == nil { + out[fields[0]] = gid + } + } + return out +} + +// GroupName is the name this machine's group database gives a number, empty when none. A variable so a test can +// stand in for groups it cannot make. +var GroupName = func(gid int) string { + if g, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil { + return g.Name + } + return "" +} + +// rootGroup is why gid is a root group here — 0, a number a root group has, or a number named as one — or "". +func rootGroup(gid int) string { + if gid == 0 { + return "group 0" + } + if name, is := RootGroupIDs()[gid]; is { + return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name) + } + if name := GroupName(gid); slices.Contains(accounts.RootGroups, name) { + return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name) + } + return "" +} + +// rootOwned is why a folder owned by uid and gid is root's data (ADR 0325 §12), or "": owned by root, or by a +// root group. +func rootOwned(uid, gid uint32) string { + if uid == 0 { + return "owned by root" + } + if why := rootGroup(int(gid)); why != "" { + return "owned by " + why + } + return "" +} + +// rootIdentity is why a module identity's numbers are refused (ADR 0325 §11), or nil. +func rootIdentity(uid, gid int) error { + if uid <= 0 { + return fmt.Errorf("its uid is %d, and a module's identity is never root (novox/hq ADR 0325 §11)", uid) + } + if why := rootGroup(gid); why != "" { + return fmt.Errorf("its gid is %s, and a module's identity is never in a root group (novox/hq ADR 0325 §11)", why) + } + return nil +} + +// Handed is every group a record that went in this apply put an account in, and another declared resource still +// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group +// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it +// there went first (novox/hq ADR 0252 §3). +type Handed map[string]map[string]bool + +func (h Handed) hand(account, group string) { + if h == nil { + return + } + if h[account] == nil { + h[account] = map[string]bool{} + } + h[account][group] = true +} + +func (h Handed) has(account, group string) bool { return h[account][group] } diff --git a/internal/apply/schedule.go b/internal/apply/schedule.go index 690e032..2d8e969 100644 --- a/internal/apply/schedule.go +++ b/internal/apply/schedule.go @@ -73,9 +73,6 @@ type scheduledJob struct { // hold is the runtime names of the containers held still for the duration of a run, in the // order the step named them (novox/hq ADR 0189). hold []string - // groups is the number of every group its read-write accesses grant it, read when it was armed (novox/hq - // ADR 0312): a fire runs with them as an apply's container would. - groups []int } // heldStillFor is the runtime names of the containers a step holds still, resolved from ids. @@ -134,14 +131,6 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { defer s.mu.Unlock() seen := map[string]bool{} - // The groups each step's read-write accesses grant (novox/hq ADR 0312), read once, only when a step is here. - var granted map[string][]int - for _, r := range d.Resources { - if c, ok := r.(*declaration.Container); ok && c.Schedule != "" { - granted = ContainerGroupsOf(d) - break - } - } for _, r := range d.Resources { c, ok := r.(*declaration.Container) if !ok || c.Schedule == "" { @@ -169,7 +158,7 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { // Nothing to read: a scheduled container may not declare restart-on — it runs to completion // on its cadence rather than staying running to be restarted — so its identity cannot // depend on another resource's content and there is nothing to pass. - spec := containerSpec(c, inputs{containerGroups: granted}) + spec := containerSpec(c, inputs{}) // The runtime stops containers by name; the declaration names them by id. Resolved here, // against the declaration this job was armed from, so a fire never has to look anything up // (novox/hq ADR 0189). The parser has already refused an id that is not a container here. @@ -178,14 +167,12 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { // Unchanged: keep where it is in its cadence, refresh the declaration pointer only. existing.container = c existing.hold = hold - existing.groups = granted[c.Identity()] continue } // New or changed: arm it for the next due minute after now. next, _ := cron.Next(s.clock.Now()) s.jobs[c.Identity()] = &scheduledJob{ id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold, - groups: granted[c.Identity()], } } @@ -295,7 +282,7 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) { // one that is not there is the state we want, so its error is ignored — the same as run-once. _, _ = s.run(ctx, cri, "rm", "-f", j.container.Name) - args := foregroundRunArgs(j.container, j.spec, j.groups) + args := foregroundRunArgs(j.container, j.spec) if _, err := s.run(ctx, cri, args...); err != nil { s.log(fmt.Sprintf( "scheduled step %s: run exited non-zero — recorded, and left for the next run "+ diff --git a/internal/apply/vocabulary_test.go b/internal/apply/vocabulary_test.go index 882d056..5fc8257 100644 --- a/internal/apply/vocabulary_test.go +++ b/internal/apply/vocabulary_test.go @@ -447,11 +447,11 @@ func TestAChangedResolverOrAddressIsAChangedContainer(t *testing.T) { base := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00"} withDns := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", Dns: []string{"192.168.203.254"}} withIP := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", IP: "192.168.203.254"} - plain := containerSpecReading(base, nil, nil, nil) - if containerSpecReading(withDns, nil, nil, nil) == plain { + plain := containerSpecReading(base, nil, nil) + if containerSpecReading(withDns, nil, nil) == plain { t.Error("adding a resolver did not change the spec, so it can never reach a running container") } - if containerSpecReading(withIP, nil, nil, nil) == plain { + if containerSpecReading(withIP, nil, nil) == plain { t.Error("adding an address did not change the spec, so it can never reach a running container") } } diff --git a/internal/apply/walk.go b/internal/apply/walk.go new file mode 100644 index 0000000..51d9d58 --- /dev/null +++ b/internal/apply/walk.go @@ -0,0 +1,66 @@ +package apply + +import ( + "errors" + "path/filepath" + "time" +) + +// A bounded walk under a kept folder (novox/hq ADR 0325 §8, §10, issue 567). +// +// **Walked by descriptor.** The folder is opened as a mesh directory is (meshWalk: no link another account could +// make, the folder itself no link), and everything under it is reached from its parent's descriptor with +// O_NOFOLLOW: a link met is passed by, a folder whose device is not the kept folder's is not entered, and an entry +// opened is checked to be the one that was looked at. An entry's ACL is read and written through its own +// descriptor, so a name swapped for a link between the look and the change changes nothing. +// +// **Bounded, and resumed.** A media library is millions of files, and a walk that never ends is a machine that +// cannot say what it did. Each walk reads at most so many entries for at most so long; one that meets its bound +// says how far it got (a cursor, the names of each level below the folder, kept in the engine's state and never +// said) and the next goes on from there, in the order of the names, so successive looks cover a folder whole. + +// WalkLimits bound one walk under a folder: how many entries it reads, and for how long. +type WalkLimits struct { + Entries int + Time time.Duration +} + +// ActLimits bound the take act and the walk that adds or drops a user's entries. One stopped goes on at the next +// apply, from the start: what already has its entries is passed quickly. +var ActLimits = WalkLimits{Entries: 5_000_000, Time: 30 * time.Minute} + +// SweepLimits bound the sweep of one kept folder on one look (ADR 0325 §10): the look must come back within its +// own bound whatever the folder holds. +var SweepLimits = WalkLimits{Entries: 2000, Time: 2 * time.Second} + +// WalkCounts is what one walk did or found. Counts only: never a path below the folder. +type WalkCounts struct { + // Folders and Files are the entries changed; Kept the ones that already had what they need. + Folders, Files, Kept int + // Links are links passed by, Other entries neither a file nor a folder, Mounts folders on another + // filesystem not entered. + Links, Other, Mounts int + // Read is every entry looked at. + Read int + // Narrowed is the entries that gave a user less than its right (judged by entry and mask together) and + // were put back; Unrepaired the ones that could not be — the owner's bits, which the mesh never changes. + Narrowed, Unrepaired int +} + +// ErrWalkBound is a walk that met its bound before it was through: what it did stays, and the cursor says where +// the next one goes on. +var ErrWalkBound = errors.New("the walk met its bound") + +// ErrNoACL is a filesystem that allows no ACLs here (ENOTSUP): a ZFS dataset whose acltype is off, say. The access +// waits for the take act, which sets it where it can (ADR 0325 §8). +var ErrNoACL = errors.New("the filesystem allows no access entries") + +// keptPath refuses a folder that is not stated exactly: absolute and clean, so `..` and a doubled separator +// never name it. +func keptPath(path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("the folder is not stated as the engine states one (absolute, no `..`, no doubled or " + + "trailing separator); nothing was changed") + } + return nil +} diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go new file mode 100644 index 0000000..692b312 --- /dev/null +++ b/internal/apply/walk_linux.go @@ -0,0 +1,572 @@ +//go:build linux + +package apply + +// The descriptor half of walk.go and acl.go (novox/hq ADR 0325 §5, §8, §9, §10). + +import ( + "errors" + "fmt" + "io" + "os" + "sort" + "time" + + "golang.org/x/sys/unix" +) + +// walkStats is how a walk reads an entry and an open descriptor: the machine's own calls, or a test's stand-in +// for a device it cannot mount. +type walkStats struct { + at func(dirfd int, name string, st *unix.Stat_t) error + fd func(fd int, st *unix.Stat_t) error +} + +var machineStats = walkStats{ + at: func(dirfd int, name string, st *unix.Stat_t) error { + return unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW) + }, + fd: unix.Fstat, +} + +// visitFunc is what a walk does to one folder or file, open on fd, whose stat is st: whether it changed it, and +// whether it found the entry narrowed and could not put it back. An error stops the walk. +type visitFunc func(fd int, st unix.Stat_t, folder bool) (changed, narrowed, unrepaired bool, err error) + +// aclWalk is one walk under a kept folder, from the descriptor of the folder itself. +type aclWalk struct { + dev uint64 + deadline time.Time + limit int + stats walkStats + visit visitFunc + counts WalkCounts + // at is the names of the levels below the folder down to the entry being walked; Cursor, once the walk met + // its bound, where the next goes on. + at []string + Cursor []string + // Failed is the first failure on one entry, in words with no path. + Failed string +} + +// openKept opens a kept folder as a mesh directory is opened (no link another account could make, the folder +// itself no link) and starts a walk from it on the folder's own device. +func openKept(path string, limits WalkLimits, stats walkStats, visit visitFunc) (*aclWalk, int, unix.Stat_t, error) { + var st unix.Stat_t + if err := keptPath(path); err != nil { + return nil, -1, st, err + } + mw := &meshWalk{want: path} + fd, err := mw.open() + if errors.Is(err, errNotThereYet) { + return nil, -1, st, errors.New("the folder is not there; nothing was changed") + } + if err != nil { + return nil, -1, st, errors.New("the folder is not walked: " + pathless(err)) + } + if mw.At != path { + unix.Close(fd) + return nil, -1, st, errors.New("the folder is reached through a link, and the engine walks no folder " + + "through one; nothing was changed") + } + if err := stats.fd(fd, &st); err != nil { + unix.Close(fd) + return nil, -1, st, fmt.Errorf("the folder cannot be read: %v; nothing was changed", err) + } + return &aclWalk{dev: uint64(st.Dev), deadline: time.Now().Add(limits.Time), limit: limits.Entries, + stats: stats, visit: visit}, fd, st, nil +} + +// walkKept walks path whole, or from resume, with visit: the walk, with what it did and the cursor where it +// stopped when it met its bound (with ErrWalkBound), and the walk's own error otherwise. +func walkKept(path string, limits WalkLimits, stats walkStats, resume []string, visit visitFunc) (*aclWalk, error) { + w, fd, st, err := openKept(path, limits, stats, visit) + if err != nil { + return &aclWalk{}, err + } + return w, w.dir(fd, st, resume) +} + +// read counts one entry, and says the bound when it is met, keeping where the walk stood as the cursor. +func (w *aclWalk) read() error { + if w.counts.Read >= w.limit || time.Now().After(w.deadline) { + w.Cursor = append([]string(nil), w.at...) + return ErrWalkBound + } + w.counts.Read++ + return nil +} + +// count adds one visit's result. +func (w *aclWalk) count(folder, changed, narrowed, unrepaired bool) { + switch { + case changed && folder: + w.counts.Folders++ + case changed: + w.counts.Files++ + default: + w.counts.Kept++ + } + if narrowed { + w.counts.Narrowed++ + } + if unrepaired { + w.counts.Unrepaired++ + } +} + +// dir is one folder, open on fd, whose stat is st: the folder itself (unless the walk resumes below it, which an +// earlier walk visited), then every entry in it in the order of their names. It closes fd. +func (w *aclWalk) dir(fd int, st unix.Stat_t, resume []string) error { + f := os.NewFile(uintptr(fd), "") + defer f.Close() + if len(resume) == 0 { + if err := w.read(); err != nil { + return err + } + if err := w.visited(fd, st, true); err != nil { + return err + } + } + names, err := f.Readdirnames(-1) + if err != nil { + return fmt.Errorf("a folder could not be read: %v", err) + } + sort.Strings(names) + for _, name := range names { + var below []string + if len(resume) > 0 { + if name < resume[0] { + continue + } + if name == resume[0] { + below = resume[1:] + } + resume = nil + } + w.at = append(w.at, name) + err := w.entry(fd, name, below) + w.at = w.at[:len(w.at)-1] + if err != nil { + return err + } + } + return nil +} + +// entry is one name in the folder open on dirfd: a link passed by, a folder on this device walked, a file +// visited through its own descriptor, anything else left. +func (w *aclWalk) entry(dirfd int, name string, resume []string) error { + var st unix.Stat_t + if err := w.stats.at(dirfd, name, &st); err != nil { + if errors.Is(err, unix.ENOENT) { + return nil + } + return fmt.Errorf("an entry could not be read: %v", err) + } + switch st.Mode & unix.S_IFMT { + case unix.S_IFLNK: + w.counts.Links++ + return nil + case unix.S_IFDIR, unix.S_IFREG: + default: + w.counts.Other++ + return nil + } + if uint64(st.Dev) != w.dev { + w.counts.Mounts++ + return nil + } + isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR + flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK + if isDir { + flags |= unix.O_DIRECTORY + } + fd, err := unix.Openat(dirfd, name, flags, 0) + switch { + case errors.Is(err, unix.ENOENT): + return nil + case errors.Is(err, unix.ELOOP), errors.Is(err, unix.ENOTDIR): + // Swapped for a link, or for something else, after it was looked at: passed by. + w.counts.Links++ + return nil + case err != nil: + return fmt.Errorf("an entry could not be opened: %v", err) + } + var opened unix.Stat_t + if err := w.stats.fd(fd, &opened); err != nil { + unix.Close(fd) + return fmt.Errorf("an entry could not be read: %v", err) + } + if opened.Dev != st.Dev || opened.Ino != st.Ino || opened.Mode&unix.S_IFMT != st.Mode&unix.S_IFMT { + unix.Close(fd) + return errors.New("an entry changed while it was walked, and the walk stopped there") + } + if isDir { + return w.dir(fd, opened, resume) + } + defer unix.Close(fd) + if err := w.read(); err != nil { + return err + } + return w.visited(fd, opened, false) +} + +// visited visits one entry and counts what it did. A filesystem that allows no ACLs stops the walk; any other +// failure on one entry (an immutable file, say) is counted unrepaired, its words kept once, and the walk goes on. +func (w *aclWalk) visited(fd int, st unix.Stat_t, folder bool) error { + changed, narrowed, unrepaired, err := w.visit(fd, st, folder) + if errors.Is(err, ErrNoACL) { + return err + } + if err != nil { + unrepaired = true + if w.Failed == "" { + w.Failed = err.Error() + } + } + w.count(folder, changed, narrowed, unrepaired) + return nil +} + +// getACL reads one of an entry's ACLs through its descriptor: the access ACL, or a folder's default. An access +// ACL that is not there is the one its mode stands for; a default that is not there is nil. +func getACL(fd int, name string, mode uint32) (acl, error) { + buf := make([]byte, 4+8*32) + for { + n, err := unix.Fgetxattr(fd, name, buf) + switch { + case errors.Is(err, unix.ENODATA): + if name == aclAccessXattr { + return minimalACL(mode), nil + } + return nil, nil + case errors.Is(err, unix.ERANGE): + buf = make([]byte, 2*len(buf)) + continue + case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP): + return nil, ErrNoACL + case err != nil: + return nil, fmt.Errorf("an entry's ACL could not be read: %v", err) + } + return decodeACL(buf[:n]) + } +} + +// setACL writes one of an entry's ACLs through its descriptor, refusing one the kernel would. +func setACL(fd int, name string, a acl) error { + if err := a.check(); err != nil { + return err + } + err := unix.Fsetxattr(fd, name, a.encode(), 0) + switch { + case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP): + return ErrNoACL + case err != nil: + return fmt.Errorf("an entry's ACL could not be written: %v", err) + } + return nil +} + +// sameACL is whether a and b hold the same entries, whatever their order. +func sameACL(a, b acl) bool { + return string(a.encode()) == string(b.encode()) +} + +// needs is, for a folder or a file, each kept uid's bits — except its owner's, which its owner's bits give and the +// mesh never changes. +func needs(users map[uint32]bool, owner uint32, folder bool) map[uint32]uint16 { + out := map[uint32]uint16{} + for uid, rw := range users { + if uid != owner { + out[uid] = needOn(rw, folder) + } + } + return out +} + +// keepVisit makes users' entries on each entry (ADR 0325 §5): the access ACL with each uid's entry and a mask +// that covers it, and on a folder the same as its default, so what is made under it inherits them. An entry that +// had them is not written. A user that owns the entry is judged by its owner's bits, never given an entry: an +// owner whose bits fall short is unrepaired, said, and never changed. +func keepVisit(users map[uint32]bool) visitFunc { + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + mode := uint32(st.Mode) & 0o7777 + need := needs(users, st.Uid, folder) + access, err := getACL(fd, aclAccessXattr, mode) + if err != nil { + return false, false, false, err + } + narrowed, unrepaired := judgeACL(access, users, st, folder) + changed := false + if kept := keptACL(access, need); !sameACL(kept, access) { + if err := setACL(fd, aclAccessXattr, kept); err != nil { + return false, narrowed, true, err + } + changed = true + } + if folder { + def, err := getACL(fd, aclDefaultXattr, mode) + if err != nil { + return changed, narrowed, unrepaired, err + } + base := def + if base == nil { + // No default yet: one whose owner's, owning group's and others' entries are the folder's own, as + // setfacl -d derives it. + base = minimalACL(mode) + if g, ok := access.perm(aclGroupObj, 0); ok { + base[1].Perm = g + } + } + // The default is for what anyone makes under the folder, never its owner alone: every user's entry. + all := map[uint32]uint16{} + for uid, rw := range users { + all[uid] = needOn(rw, true) + } + if kept := keptACL(base, all); def == nil || !sameACL(kept, def) { + if err := setACL(fd, aclDefaultXattr, kept); err != nil { + return changed, narrowed, true, err + } + changed = true + } + } + return changed, narrowed, unrepaired, nil + } +} + +// judgeACL is whether an entry with access ACL a gives a user less than its right — by entry and mask together +// (ADR 0325 §6) — and whether that is one the mesh cannot put back: the owner's own bits. +func judgeACL(a acl, users map[uint32]bool, st unix.Stat_t, folder bool) (narrowed, unrepaired bool) { + mode := uint32(st.Mode) & 0o7777 + for uid, rw := range users { + need := needOn(rw, folder) + if effective(a, uid, st.Uid, mode)&need == need { + continue + } + narrowed = true + if uid == st.Uid { + unrepaired = true + } + } + return narrowed, unrepaired +} + +// dropVisit takes exactly the entries of each uid of drop out of each entry's access ACL and a folder's default +// (ADR 0325 §8): no other entry, and not the mask, so nobody else loses a right. +func dropVisit(drop map[uint32]bool) visitFunc { + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + mode := uint32(st.Mode) & 0o7777 + changed := false + for _, name := range []string{aclAccessXattr, aclDefaultXattr} { + if name == aclDefaultXattr && !folder { + continue + } + a, err := getACL(fd, name, mode) + if errors.Is(err, ErrNoACL) { + continue + } + if err != nil { + // Counted unrepaired and said by the walk, which then walks it again: never read as "nothing to drop". + return changed, false, true, err + } + if a == nil { + continue + } + if left := withoutUsers(a, drop); len(left) != len(a) { + if err := setACL(fd, name, left); err != nil { + return changed, false, true, err + } + changed = true + } + } + return changed, false, false, nil + } +} + +// dropThenKeep is drop then keep, on one walk: a user whose use changed, or one gone beside one come. +func dropThenKeep(drop map[uint32]bool, users map[uint32]bool) visitFunc { + d, k := dropVisit(drop), keepVisit(users) + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + c1, _, _, err := d(fd, st, folder) + if err != nil { + return c1, false, false, err + } + c2, n, u, err := k(fd, st, folder) + return c1 || c2, n, u, err + } +} + +// folderEmpty is whether the folder open on fd holds nothing yet: the data the mesh may keep without asking. +func folderEmpty(fd int) (bool, error) { + dup, err := unix.Dup(fd) + if err != nil { + return false, err + } + f := os.NewFile(uintptr(dup), "") + defer f.Close() + names, err := f.Readdirnames(1) + switch { + case len(names) > 0: + return false, nil + case err == nil, errors.Is(err, io.EOF): + return true, nil + } + return false, fmt.Errorf("the folder could not be read: %v", err) +} + +// keepTop keeps users' entries on the kept folder itself (ADR 0325 §5): what every apply does, beside the walks. +func keepTop(path string, users map[uint32]bool) (changed bool, err error) { + w, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, keepVisit(users)) + if err != nil { + return false, err + } + defer unix.Close(fd) + changed, _, _, err = w.visit(fd, st, true) + return changed, err +} + +// FolderFacts is what the engine reads of a folder before keeping it: its owner, group and whether it holds data. +type FolderFacts struct { + UID, GID uint32 + Empty bool +} + +// readFolder reads a folder's facts through the descriptor a mesh directory is opened by. +func readFolder(path string) (FolderFacts, error) { + if err := keptPath(path); err != nil { + return FolderFacts{}, err + } + mw := &meshWalk{want: path} + fd, err := mw.open() + if err != nil { + return FolderFacts{}, err + } + defer unix.Close(fd) + if mw.At != path { + return FolderFacts{}, errors.New("the folder is reached through a link, and the engine keeps no folder " + + "through one") + } + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return FolderFacts{}, err + } + empty, err := folderEmpty(fd) + if err != nil { + return FolderFacts{}, err + } + return FolderFacts{UID: st.Uid, GID: st.Gid, Empty: empty}, nil +} + +// CreatePath makes a missing accessed folder (ADR 0325 §9): each missing level made from the nearest existing +// ancestor down, each with that ancestor's owner, group and mode. No link is followed (the ancestor is reached as +// a mesh directory is, and each level is opened with O_NOFOLLOW), no level is made on another filesystem, no file +// is made, and nothing that was there is changed: a level that appears meanwhile stops it. The caller has judged +// that the ancestor is on the placed filesystem and not root's. How many levels it made. +func CreatePath(path string) (int, error) { + return createPath(path, machineStats) +} + +func createPath(path string, stats walkStats) (int, error) { + m, err := NearestAncestor(path) + if err != nil { + return 0, fmt.Errorf("the folder cannot be made: %v; nothing was changed", err) + } + if !m.Info.IsDir() { + return 0, errors.New("its nearest existing parent is not a folder (a file or a link), and a folder is " + + "made only below a folder; nothing was changed") + } + mw := &meshWalk{want: m.Ancestor} + fd, err := mw.open() + if err != nil { + return 0, fmt.Errorf("the nearest existing parent is not walked: %v; nothing was changed", err) + } + if mw.At != m.Ancestor { + unix.Close(fd) + return 0, errors.New("the nearest existing parent is reached through a link, and nothing is made " + + "through one; nothing was changed") + } + var anc unix.Stat_t + if err := stats.fd(fd, &anc); err != nil { + unix.Close(fd) + return 0, fmt.Errorf("the nearest existing parent cannot be read: %v; nothing was changed", err) + } + if why := rootOwned(anc.Uid, anc.Gid); why != "" { + unix.Close(fd) + return 0, fmt.Errorf("its nearest existing parent is %s, and nothing is made under root's data "+ + "(novox/hq ADR 0325 §12); nothing was changed", why) + } + mode := uint32(anc.Mode) & 0o7777 + made := 0 + for _, name := range m.Levels { + // Made for its maker alone (the engine, as root), so it is nobody else's until it is the ancestor's. + if err := unix.Mkdirat(fd, name, 0o700); err != nil { + unix.Close(fd) + if errors.Is(err, unix.EEXIST) { + return made, errors.New("a level of the folder appeared while it was made, and nothing that is " + + "there is changed; it stopped there") + } + return made, fmt.Errorf("a level could not be made: %v", err) + } + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + unix.Close(fd) + if err != nil { + return made, fmt.Errorf("a level made could not be opened: %v", err) + } + fd = next + made++ + var st unix.Stat_t + if err := stats.fd(fd, &st); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be read: %v", err) + } + if uint64(st.Dev) != uint64(anc.Dev) { + unix.Close(fd) + return made, errors.New("a level made is on another filesystem than its parent, and it stopped there") + } + if st.Uid != anc.Uid || st.Gid != anc.Gid { + if err := unix.Fchown(fd, int(anc.Uid), int(anc.Gid)); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be given its parent's owner: %v", err) + } + } + if err := unix.Fchmod(fd, mode); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be given its parent's mode: %v", err) + } + } + unix.Close(fd) + return made, nil +} + +// MountFsid is the kernel's id of the filesystem at point (statfs f_fsid), in hex; empty when it cannot be read. +// A variable so a test can stand in for a filesystem it cannot mount. +var MountFsid = func(point string) string { + var st unix.Statfs_t + if err := unix.Statfs(point, &st); err != nil { + return "" + } + return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1])) +} + +// MountUUID is the UUID of the filesystem at point when it is on a block device: the name in /dev/disk/by-uuid of +// the device the mount's own device number is. Empty for one on no block device (NFS, FUSE, ZFS, a btrfs +// subvolume's own device). A variable so a test can stand in for disks it cannot make. +var MountUUID = func(point string) string { + var st unix.Stat_t + if err := unix.Stat(point, &st); err != nil { + return "" + } + const dir = "/dev/disk/by-uuid" + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + for _, e := range entries { + var dev unix.Stat_t + if unix.Stat(dir+"/"+e.Name(), &dev) == nil && dev.Mode&unix.S_IFMT == unix.S_IFBLK && dev.Rdev == st.Dev { + return e.Name() + } + } + return "" +} diff --git a/internal/apply/walk_other.go b/internal/apply/walk_other.go new file mode 100644 index 0000000..7d9c79a --- /dev/null +++ b/internal/apply/walk_other.go @@ -0,0 +1,61 @@ +//go:build !linux + +package apply + +import ( + "context" + "errors" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The node-engine runs on Linux: a folder is kept only by descriptor and through the kernel's ACL attributes, +// which this system has not. Everything refuses, saying so, and changes nothing. + +var errNoKeeping = errors.New("a shared folder is kept only by descriptor, which this system cannot; nothing was changed") + +// FolderFacts is what the engine reads of a folder before keeping it. +type FolderFacts struct { + UID, GID uint32 + Empty bool +} + +func readFolder(string) (FolderFacts, error) { return FolderFacts{}, errNoKeeping } +func keepTop(string, map[uint32]bool) (bool, error) { return false, errNoKeeping } +func CreatePath(string) (int, error) { return 0, errNoKeeping } +func ReconcileKept(context.Context) []string { return nil } +func JudgeKept(string, []store.KeptUser, WalkLimits) (WalkCounts, error) { + return WalkCounts{}, errNoKeeping +} + +// KeptLook is what one look found of a kept folder. +type KeptLook struct { + Short []string + Busy bool + Sweep store.KeptSweep + Repaired, Unrepaired int + Failed string +} + +func LookKept(string, []store.KeptUser, WalkLimits) KeptLook { + return KeptLook{Failed: errNoKeeping.Error()} +} + +func TakeFolder(context.Context, *declaration.Declaration, string, string, string, string, Runner) (<-chan struct{}, + string, error) { + return nil, "", errNoKeeping +} + +// Watcher watches nothing where a folder is not kept. +type Watcher struct{} + +func NewWatcher() (*Watcher, error) { return nil, errNoKeeping } +func (*Watcher) Set(map[string][]store.KeptUser) {} +func (*Watcher) Run(context.Context) {} + +// MountFsid reads no filesystem id where a folder is not kept. +var MountFsid = func(string) string { return "" } + +// MountUUID reads no UUID where a folder is not kept. +var MountUUID = func(string) string { return "" } diff --git a/internal/apply/watch_linux.go b/internal/apply/watch_linux.go new file mode 100644 index 0000000..8bbb43f --- /dev/null +++ b/internal/apply/watch_linux.go @@ -0,0 +1,350 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "strconv" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/store" +) + +// The watch on each kept folder (novox/hq ADR 0325 §6). +// +// What an identity makes in a kept folder inherits the default entries, and its mask follows the mode its maker +// asked for: a file made at 0644 gives a read-write user only read until its mask is widened. The engine watches +// every folder of each kept tree (inotify: made, moved in, attributes changed) and, on notice, gives the entry its +// users' entries and a mask that covers them — the same keepVisit the walks use, through the entry's own +// descriptor, reached from the kept folder down by O_NOFOLLOW, never through a link and never on another +// filesystem. Notice can be lost (the kernel's queue overflowed) and watches run out (the machine's limit, ENOSPC): +// then the record says so and the sweep finds what was made where no watch is (§10). + +// watchMask is what a watch hears: an entry made or moved in, and an entry's mode or ACL changed. +// It is added through /proc/self/fd, whose link is the folder's own descriptor: following it is what reaches the +// folder that was opened, so the watch takes no IN_DONT_FOLLOW. +const watchMask = unix.IN_CREATE | unix.IN_MOVED_TO | unix.IN_ATTRIB | unix.IN_ONLYDIR + +// MaxWatchedPerFolder bounds the folders watched under one kept folder: beyond it the sweep alone keeps the rest. +var MaxWatchedPerFolder = 100_000 + +// Watcher watches every kept folder's tree. +type Watcher struct { + mu sync.Mutex + fd int + dirs map[int]watchedDir + roots map[string]*watchedRoot +} + +// watchedDir is one watched folder: its kept folder and the names below it. +type watchedDir struct { + root string + rel []string +} + +// watchedRoot is one kept folder watched: its users, its device, and its watches. +type watchedRoot struct { + users map[uint32]bool + dev uint64 + wds []int + out bool +} + +// NewWatcher opens the machine's notice of changes. +func NewWatcher() (*Watcher, error) { + // Not blocking: Run polls it, so the end of its context ends it whatever the kernel has to say. + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if err != nil { + return nil, err + } + return &Watcher{fd: fd, dirs: map[int]watchedDir{}, roots: map[string]*watchedRoot{}}, nil +} + +// Set is every kept folder to watch, with its users: one newly kept is watched whole (within +// MaxWatchedPerFolder), one no longer kept is let go, and one whose users changed is repaired with the new ones. +func (w *Watcher) Set(folders map[string][]store.KeptUser) { + w.mu.Lock() + defer w.mu.Unlock() + for path, root := range w.roots { + if _, still := folders[path]; !still { + for _, wd := range root.wds { + _, _ = unix.InotifyRmWatch(w.fd, uint32(wd)) + delete(w.dirs, wd) + } + delete(w.roots, path) + } + } + for path, users := range folders { + if root, had := w.roots[path]; had { + root.users = usersOf(users) + continue + } + root := &watchedRoot{users: usersOf(users)} + w.roots[path] = root + w.watchTree(path, root) + w.say(path, root) + } +} + +// say records how many folders of a kept folder are watched, and whether the watches ran out. +func (w *Watcher) say(path string, root *watchedRoot) { + n, out := len(root.wds), root.out + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Watches, f.OutOfWatches = n, out + return had + }) +} + +// watchTree watches a kept folder and every folder under it on its device, by descriptor: each watch is added +// through /proc/self/fd of the folder's own descriptor, so no name is followed to another place. +func (w *Watcher) watchTree(path string, root *watchedRoot) { + _, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return + } + root.dev = uint64(st.Dev) + w.watchBelow(path, root, fd, nil) +} + +// watchBelow watches the folder open on fd, rel below root, and every folder under it. It closes fd. +func (w *Watcher) watchBelow(path string, root *watchedRoot, fd int, rel []string) { + defer unix.Close(fd) + if root.out || len(root.wds) >= MaxWatchedPerFolder { + root.out = true + return + } + wd, err := unix.InotifyAddWatch(w.fd, "/proc/self/fd/"+strconv.Itoa(fd), watchMask) + if errors.Is(err, unix.ENOSPC) { + root.out = true + return + } + if err != nil { + return + } + root.wds = append(root.wds, wd) + w.dirs[wd] = watchedDir{root: path, rel: append([]string(nil), rel...)} + dup, err := unix.Dup(fd) + if err != nil { + return + } + names, err := readNames(dup) + if err != nil { + return + } + for _, name := range names { + var st unix.Stat_t + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR || + uint64(st.Dev) != root.dev { + continue + } + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + continue + } + w.watchBelow(path, root, next, append(rel, name)) + } +} + +// watchPoll is how long Run waits for notice before it looks whether its context ended. +const watchPoll = 250 * time.Millisecond + +// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder, and then closes the +// watch: polled, so a quiet machine's watch ends as soon as a busy one's. +func (w *Watcher) Run(ctx context.Context) { + defer unix.Close(w.fd) + buf := make([]byte, 64*1024) + for ctx.Err() == nil { + ready, err := unix.Poll([]unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}}, int(watchPoll/time.Millisecond)) + if err != nil && !errors.Is(err, unix.EINTR) { + return + } + if ready <= 0 { + continue + } + n, err := unix.Read(w.fd, buf) + if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR) { + continue + } + if err != nil || n <= 0 { + return + } + for at := 0; at+unix.SizeofInotifyEvent <= n; { + ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[at])) + nameAt := at + unix.SizeofInotifyEvent + name := cString(buf[nameAt : nameAt+int(ev.Len)]) + at = nameAt + int(ev.Len) + if ev.Mask&unix.IN_Q_OVERFLOW != 0 { + w.lost() + continue + } + if ev.Mask&unix.IN_IGNORED != 0 { + // The watch is gone — its folder removed, or let go by Set: what pointed at it goes too. + w.forget(int(ev.Wd)) + continue + } + w.heard(int(ev.Wd), name, ev.Mask) + } + } +} + +// lost is notice lost (the kernel's queue overflowed): every kept folder is watched afresh, and what was made +// meanwhile is the sweep's. +func (w *Watcher) lost() { + w.mu.Lock() + defer w.mu.Unlock() + for path, root := range w.roots { + for _, wd := range root.wds { + _, _ = unix.InotifyRmWatch(w.fd, uint32(wd)) + delete(w.dirs, wd) + } + root.wds, root.out = nil, false + w.watchTree(path, root) + w.say(path, root) + } +} + +// forget lets go of a watch the kernel removed. +func (w *Watcher) forget(wd int) { + w.mu.Lock() + defer w.mu.Unlock() + dir, ok := w.dirs[wd] + if !ok { + return + } + delete(w.dirs, wd) + if root := w.roots[dir.root]; root != nil { + for i, had := range root.wds { + if had == wd { + root.wds = append(root.wds[:i], root.wds[i+1:]...) + break + } + } + w.say(dir.root, root) + } +} + +// watched is how many folders are watched now. +func (w *Watcher) watched() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.dirs) +} + +// heard repairs one entry a watch heard of: name in the watched folder wd, or the folder itself. +func (w *Watcher) heard(wd int, name string, mask uint32) { + w.mu.Lock() + dir, ok := w.dirs[wd] + root := w.roots[dir.root] + var users map[uint32]bool + var dev uint64 + if root != nil { + // Read under the lock: Set gives a root new users while notices are heard. + users, dev = root.users, root.dev + } + w.mu.Unlock() + if !ok || root == nil { + return + } + fd, err := openBelow(dir.root, dir.rel, dev) + if err != nil { + return + } + defer unix.Close(fd) + visit := keepVisit(users) + if name == "" { + var st unix.Stat_t + if unix.Fstat(fd, &st) == nil { + _, _, _, _ = visit(fd, st, true) + } + return + } + var st unix.Stat_t + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != dev { + return + } + isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR + if !isDir && st.Mode&unix.S_IFMT != unix.S_IFREG { + return + } + flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK + if isDir { + flags |= unix.O_DIRECTORY + } + efd, err := unix.Openat(fd, name, flags, 0) + if err != nil { + return + } + var opened unix.Stat_t + if unix.Fstat(efd, &opened) != nil || opened.Ino != st.Ino || opened.Dev != st.Dev { + unix.Close(efd) + return + } + _, _, _, _ = visit(efd, opened, isDir) + if isDir && mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 { + // A folder made or moved in is watched, and what was moved in with it is the sweep's. + w.mu.Lock() + defer w.mu.Unlock() + if w.roots[dir.root] != root { + // Let go by Set meanwhile: nothing more is watched for it. + unix.Close(efd) + return + } + w.watchBelow(dir.root, root, efd, append(append([]string(nil), dir.rel...), name)) + w.say(dir.root, root) + return + } + unix.Close(efd) +} + +// openBelow opens the folder rel below the kept folder root, each level by O_NOFOLLOW on root's device. +func openBelow(root string, rel []string, dev uint64) (int, error) { + _, fd, _, err := openKept(root, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return -1, err + } + for _, name := range rel { + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + unix.Close(fd) + if err != nil { + return -1, err + } + var st unix.Stat_t + if err := unix.Fstat(next, &st); err != nil || uint64(st.Dev) != dev { + unix.Close(next) + return -1, errors.New("a level is on another filesystem") + } + fd = next + } + return fd, nil +} + +// readNames reads every name in the folder open on fd, and closes it. +func readNames(fd int) ([]string, error) { + var names []string + buf := make([]byte, 32*1024) + defer unix.Close(fd) + for { + n, err := unix.ReadDirent(fd, buf) + if err != nil { + return names, err + } + if n <= 0 { + return names, nil + } + _, _, names = unix.ParseDirent(buf[:n], -1, names) + } +} + +func cString(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index e2f6d29..4f4ba92 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -456,8 +456,10 @@ const ( // **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own — // it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A // media library, a download spool is the operator's: it existed before the mesh, several modules -// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The -// host confirms it is there and mounts it; nothing else. +// read and write it at once, and the mesh must not chown, chmod or remove it. The host confirms it +// is there and mounts it, and keeps it usable by the identities that use it through ACL entries it +// adds and nothing else (novox/hq ADR 0325 §5); a missing one is made only below a parent on the +// filesystem the data was placed on (§9). type Access struct { ID string `json:"id"` Type Type `json:"type"` @@ -465,33 +467,44 @@ type Access struct { // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` - // Grant is who the path's own group is granted to, for a read-write access (novox/hq ADR 0312 §1): the - // identities the module's code runs as. The controller says who; the node-engine reads which group owns - // the path when it applies, because only the machine knows, and never changes the path (§2). Absent is a - // controller older than the grant, or an access that grants nothing — and a read access always is one (§3). - Grant *AccessGrant `json:"grant,omitempty"` + // Users are the identities that use the folder, resolved by the controller (novox/hq ADR 0325 §3, §4): each + // module identity whose container mounts it, and the tool runner's account standing for every module whose + // tools use it. The engine keeps one ACL entry per user on the folder and as its default (§5); several + // accesses naming one folder are kept as one, each uid at the widest use any of them gives it. Absent is a + // controller older than ADR 0325, or an access nobody but others' bits reach (§2): nothing is kept. + Users []AccessUser `json:"users,omitempty"` + // Taken is the controller saying the operator's warrant for this folder was carried out (§8): the folder is + // kept although it holds data. The engine also keeps a folder it took itself, on the act, or found empty. + Taken bool `json:"taken,omitempty"` + + // Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted + // so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5). + Grant json.RawMessage `json:"grant,omitempty"` } -// AccessGrant is the identities a read-write access grants its path's group to (novox/hq ADR 0312 §1). -type AccessGrant struct { - // Containers are the module's containers that mount the path, by id: each is created with the group as a - // supplementary group of its process, and recreated when the group moves (§6). - Containers []string `json:"containers,omitempty"` - // Account is the account that runs the module's tools: put in the group as an account group, as ADR 0252 - // §1 puts one, and taken out again as §3 takes back (§5). Empty for a module whose code runs nowhere as - // an account. - Account string `json:"account,omitempty"` +// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4). +const ( + FromContainer = "container" + FromToolRunner = "tool-runner" +) + +// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its +// use, and where it runs. +type AccessUser struct { + Identity string `json:"identity"` + UID int `json:"uid"` + Use string `json:"use"` + From string `json:"from"` } -// UnmarshalJSON reads a grant as strictly as a resource is read: a field the engine does not know is a -// thing the controller believes it asked for, and is refused rather than ignored. -func (g *AccessGrant) UnmarshalJSON(raw []byte) error { - type plain AccessGrant +// UnmarshalJSON reads a user as strictly as a resource is read. +func (u *AccessUser) UnmarshalJSON(raw []byte) error { + type plain AccessUser var p plain if err := strictDecode(raw, &p); err != nil { - return fmt.Errorf("a grant: %w", err) + return fmt.Errorf("an access's user: %w", err) } - *g = AccessGrant(p) + *u = AccessUser(p) return nil } @@ -511,27 +524,41 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } - if a.Grant != nil && a.Mode != AccessReadWrite { - // A read access reads as any account may, and gains no group (novox/hq ADR 0312 §3). - problems = append(problems, where+": a read access grants nothing, and this one names a grant; "+ - "only a read-write access grants its path's group (novox/hq ADR 0312 §3)") - } - if a.Grant != nil { - for _, id := range a.Grant.Containers { - if id == "" { - problems = append(problems, where+": a grant names a container with no id") - } + for _, u := range a.Users { + switch { + case u.Identity == "" || !accountName.MatchString(u.Identity): + problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one", + where, u.Identity)) + case u.UID <= 0: + // Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a + // number below it is no account. + problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+ + "root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID)) } - if a.Grant.Account != "" && !accountName.MatchString(a.Grant.Account) { - // Handed to id, usermod and gpasswd as an argument: a name starting with a dash would be read as an - // option by each of them. - problems = append(problems, fmt.Sprintf("%s: a grant's account is an account's name — letters, digits, "+ - "dots, underscores and dashes, not starting with a dash — and %q is not one", where, a.Grant.Account)) + if u.Use != AccessRead && u.Use != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where, + u.Identity, u.Use, AccessRead, AccessReadWrite)) + } + if u.Use == AccessReadWrite && a.Mode != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where, + u.Identity, orRead(a.Mode))) + } + if u.From != FromContainer && u.From != FromToolRunner { + problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where, + u.Identity, u.From, FromContainer, FromToolRunner)) } } return problems } +// orRead is an access's mode as said: absent narrows to read. +func orRead(mode string) string { + if mode == "" { + return AccessRead + } + return mode +} + func (u *User) Identity() string { return u.ID } func (u *User) Kind() Type { return TypeUser } func (u *User) Target() string { return u.Name } @@ -1207,6 +1234,78 @@ type Container struct { // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. Health *Health `json:"health,omitempty"` + + // RunAs is who the container runs as, resolved by the controller from the module's identity and the + // image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and + // PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the + // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because + // a resource's Identity is its id. + RunAs *ContainerIdentity `json:"identity,omitempty"` +} + +// The ways an image is told who to run as (novox/hq ADR 0325 §2). +const ( + IdentityByEnv = "env" + IdentityByRun = "run" +) + +// ContainerIdentity is who a container runs as, and how its image is told. +type ContainerIdentity struct { + // Mode is IdentityByEnv or IdentityByRun. + Mode string `json:"mode"` + // UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for + // IdentityByEnv: "PUID", "PGID". + UserEnv string `json:"user_env,omitempty"` + GroupEnv string `json:"group_env,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// UnmarshalJSON reads an identity as strictly as a resource is read. +func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error { + type plain ContainerIdentity + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("a container's identity: %w", err) + } + *i = ContainerIdentity(p) + return nil +} + +// envName is a variable's name as a runtime takes it. +var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (i *ContainerIdentity) problems(where string, env map[string]string) []string { + if i == nil { + return nil + } + var problems []string + switch i.Mode { + case IdentityByEnv: + if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) { + problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+ + "it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv)) + } + for _, name := range []string{i.UserEnv, i.GroupEnv} { + if _, set := env[name]; set && name != "" { + problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+ + "for one variable", where, name)) + } + } + case IdentityByRun: + if i.UserEnv != "" || i.GroupEnv != "" { + problems = append(problems, where+": an identity told by the runtime's user option names no variable") + } + default: + problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where, + IdentityByEnv, IdentityByRun, i.Mode)) + } + // Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started. + if i.UID <= 0 || i.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+ + "never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID)) + } + return problems } func (c *Container) Identity() string { return c.ID } @@ -1289,6 +1388,7 @@ func (c *Container) validate(where string, _ bool) []string { problems = append(problems, where+": networks is for a container that keeps running; a step "+ "runs and exits, and joins nothing afterwards") } + problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) return append(problems, checkImage(where, c.Image)...) } @@ -1439,6 +1539,53 @@ type Declaration struct { // the generation of what it puts back, which may be older than what the machine applied, and is // not refused for it. PutBack bool + + // Identities is the identity of every module assigned to this machine (novox/hq ADR 0325 §3): an account + // and a group `mesh-` with the numbers the controller allocated once for the module, the same on + // every machine. The engine makes each before any resource is applied, and removes what it made of one no + // longer declared. Absent is a controller older than ADR 0325. + Identities []Identity +} + +// Identity is one module's identity, as the controller allocated it. +type Identity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// IdentityPrefix starts every module identity's name. +const IdentityPrefix = "mesh-" + +// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number +// twice — two modules with one account would write as each other. +func checkIdentities(ids []Identity) []string { + var problems []string + names, uids, gids := map[string]bool{}, map[int]string{}, map[int]string{} + for _, id := range ids { + where := fmt.Sprintf("identity %q", id.Name) + if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) { + problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s, and this one is "+ + "for the module %q", where, IdentityPrefix, id.Module)) + } + if id.UID <= 0 || id.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+ + "nor a number below it (novox/hq ADR 0325 §11); nothing of this declaration was made", where, + id.UID, id.GID)) + } + if names[id.Name] { + problems = append(problems, where+": declared twice") + } + if other, used := uids[id.UID]; used { + problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other)) + } + if other, used := gids[id.GID]; used { + problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, id.GID, other)) + } + names[id.Name], uids[id.UID], gids[id.GID] = true, id.Name, id.Name + } + return problems } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1608,6 +1755,9 @@ type envelope struct { // sends them only to a host whose reports say `reads_generation`. Generation int64 `json:"generation,omitempty"` PutBack bool `json:"put_back,omitempty"` + // Identities is optional on the wire too (novox/hq ADR 0325 §3). **An older host refuses this key**; a + // controller sends it only to a host whose health says link.IdentityContract. + Identities []Identity `json:"identities,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1625,8 +1775,9 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} - var problems []string + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, + Identities: env.Identities} + problems := checkIdentities(env.Identities) if env.Generation < 0 { // Below zero would read as "none claimed" and pass every refusal of an older generation. problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ @@ -1715,7 +1866,6 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkWhileStopped(d.Resources)...) - problems = append(problems, checkGrants(d.Resources)...) problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) problems = append(problems, checkReplaces(d.Resources)...) if env.Adoption == nil { @@ -1780,36 +1930,10 @@ func checkWhileStopped(resources []Resource) []string { return problems } -// accountName is what a grant's account may be: the portable account name the user tools take, optionally ending -// in `$`, never starting with a dash. +// accountName is what an identity's or an access user's name may be: the portable account name the user tools +// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument. var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) -// checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it -// names is a container of this declaration, as a maintenance window's are. A grant to a container the machine -// is not told about would be a group given to nothing, said as granted. -func checkGrants(resources []Resource) []string { - containers := map[string]bool{} - for _, r := range resources { - if r.Kind() == TypeContainer { - containers[r.Identity()] = true - } - } - var problems []string - for _, r := range resources { - a, ok := r.(*Access) - if !ok || a.Grant == nil { - continue - } - for _, id := range a.Grant.Containers { - if id != "" && !containers[id] { - problems = append(problems, fmt.Sprintf("resource %q: its grant names %q, and this declaration "+ - "has no container by that id", a.ID, id)) - } - } - } - return problems -} - func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. diff --git a/internal/declaration/identity_test.go b/internal/declaration/identity_test.go new file mode 100644 index 0000000..e97225f --- /dev/null +++ b/internal/declaration/identity_test.go @@ -0,0 +1,61 @@ +package declaration + +import ( + "strings" + "testing" +) + +const anImage = "docker.io/library/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +// The fields ADR 0325 adds, read as the controller sends them (novox/hq ADR 0325 §2, §3, §5, issue 567). +func TestIdentitiesUsersAndARunAsAreReadAsSent(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1, + "identities":[{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}], + "resources":[ + {"id":"radarr.app","type":"container","name":"radarr","image":"` + anImage + `", + "identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":61001}}, + {"id":"plex.app","type":"container","name":"plex","image":"` + anImage + `","identity":{"mode":"run","uid":61002,"gid":61002}}, + {"id":"radarr.movies","type":"access","path":"/srv/media/movies","mode":"read-write","taken":true, + "users":[{"identity":"mesh-radarr","uid":61001,"use":"read-write","from":"container"}, + {"identity":"nox","uid":61003,"use":"read","from":"tool-runner"}]}, + {"id":"old.movies","type":"access","path":"/srv/media/movies","mode":"read-write","grant":{"account":"nox"}}]}`)) + if err != nil { + t.Fatal(err) + } + if len(d.Identities) != 1 || d.Identities[0].UID != 61001 { + t.Fatalf("identities: %+v", d.Identities) + } + if c := d.Resources[0].(*Container); c.RunAs == nil || c.RunAs.UserEnv != "PUID" { + t.Fatalf("a container's identity: %+v", c.RunAs) + } + if a := d.Resources[2].(*Access); !a.Taken || len(a.Users) != 2 || a.Users[1].From != FromToolRunner { + t.Fatalf("an access's users: %+v", a) + } +} + +// What the engine refuses on arrival: a root number, an identity not named for its module, a field it does not +// know, a user that writes a read access, a variable set twice. +func TestWhatADeclarationMayNotSayOfAnIdentity(t *testing.T) { + for want, raw := range map[string]string{ + "never root": `"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]`, + "named mesh-": `"identities":[{"module":"m","name":"m","uid":61001,"gid":61001}],` + + `"resources":[]`, + "is mesh-a's too": `"identities":[{"module":"a","name":"mesh-a","uid":61001,"gid":61001},` + + `{"module":"b","name":"mesh-b","uid":61001,"gid":61002}],"resources":[]`, + "unknown field": `"identities":[{"module":"m","name":"mesh-m","uid":61001,"gid":61001,"home":"/"}],` + + `"resources":[]`, + "writes an access declared read": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","mode":"read",` + + `"users":[{"identity":"mesh-m","uid":61001,"use":"read-write","from":"container"}]}]`, + "uid 0": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","users":[{"identity":"root","uid":0,` + + `"use":"read","from":"container"}]}]`, + "one value": `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"env":{"PUID":"1000"},"identity":{"mode":"env","user_env":"PUID","uid":61001,"gid":61001}}]`, + `"env" or "run"`: `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"identity":{"mode":"sudo","uid":61001,"gid":61001}}]`, + } { + _, err := Parse([]byte(`{"declaration":1,` + raw + `}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("want a refusal saying %q, got %v", want, err) + } + } +} diff --git a/internal/link/access_act_test.go b/internal/link/access_act_test.go new file mode 100644 index 0000000..3d745b2 --- /dev/null +++ b/internal/link/access_act_test.go @@ -0,0 +1,68 @@ +package link + +import ( + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" +) + +// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a +// signing context of its own. The controller's side holds the same names and words. +func TestTheTakeActsAskKeepsItsFieldNamesAndContext(t *testing.T) { + body, _ := json.Marshal(AccessActAsk{Act: ActTake, Access: "radarr.movies", Warrant: "w"}) + if got := keysOf(t, body); got != "access act warrant" { + t.Fatalf("the act's fields are %q", got) + } + if AskAccessActSubject("ace") != "mesh.node.ace.ask.access-act" { + t.Fatalf("the subject is %q", AskAccessActSubject("ace")) + } + if AccessActContext != "novox-mesh access-act v2\n" { + t.Fatalf("the signing context is %q", AccessActContext) + } +} + +// signedAct is an act's ask as the controller makes it: the hand-over's fields beside the act's, signed over the +// act's context and its bytes. +func signedAct(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, act AccessActAsk) []byte { + t.Helper() + raw, _ := json.Marshal(struct { + HandOverAsk + AccessActAsk + }{ask, act}) + body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(AccessActContext), raw...))}) + return body +} + +// **Nothing is done without a warrant**, nor for any act but take, nor signed as a hand-over (ADR 0325 §8). +func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) { + m, key := aMember(t) + now := time.Now() + var given []AccessAct + handle := func(a AccessAct) (string, error) { given = append(given, a); return "started", nil } + check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}} + ask := func(nonce string) HandOverAsk { + return HandOverAsk{Node: m.Node, Path: "/srv/media/movies", By: "the operator", Expires: now.Add(HandOverGood), + Nonce: nonce} + } + for why, body := range map[string][]byte{ + "no warrant": signedAct(t, key, ask(aNonce+"01"), AccessActAsk{Act: ActTake, Access: "radarr.movies"}), + "another act": signedAct(t, key, ask(aNonce+"02"), AccessActAsk{Act: "share", Access: "radarr.movies", Warrant: "w"}), + "no access": signedAct(t, key, ask(aNonce+"03"), AccessActAsk{Act: ActTake, Warrant: "w"}), + "a hand-over's": signedAsk(t, key, ask(aNonce+"04")), + "a path as access": signedAct(t, key, ask(aNonce+"05"), AccessActAsk{Act: ActTake, Access: "/srv", Warrant: "w"}), + } { + if a := AnswerAccessAct(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was changed") { + t.Errorf("%s: not refused, or refused without saying nothing was changed: %+v", why, a) + } + } + if len(given) != 0 { + t.Fatalf("a refused ask reached the act: %+v", given) + } + a := AnswerAccessAct(check, handle, signedAct(t, key, ask(aNonce+"06"), AccessActAsk{Act: ActTake, + Access: "radarr.movies", Warrant: "w-1"})) + if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" { + t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given) + } +} diff --git a/internal/link/handover.go b/internal/link/handover.go index 1d0dd2f..69eec73 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -136,20 +136,27 @@ func AnswerHandOver(check HandOverCheck, handle HandOverFunc, body []byte) HandO // one short line naming who asked, and a path when withPath and none otherwise. The nonce is taken last, after // every other check passed. It answers the ask with who asked trimmed, or why it is refused. func believed(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, string) { + ask, _, refused := believedBytes(check, context, withPath, body) + return ask, refused +} + +// believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's +// fields: read out of them only once they are believed. +func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) { if len(body) > MaxHandOverAsk { - return HandOverAsk{}, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body)) + return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body)) } var signed SignedHandOver if err := json.Unmarshal(body, &signed); err != nil { - return HandOverAsk{}, "the ask is not a signed ask: " + err.Error() + return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error() } if len(check.Signer) != ed25519.PublicKeySize || !ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) { - return HandOverAsk{}, "the ask is not signed by the mesh" + return HandOverAsk{}, nil, "the ask is not signed by the mesh" } var ask HandOverAsk if err := json.Unmarshal(signed.Ask, &ask); err != nil { - return HandOverAsk{}, "the signed ask is not an ask: " + err.Error() + return HandOverAsk{}, nil, "the signed ask is not an ask: " + err.Error() } now := time.Now() if check.Now != nil { @@ -163,25 +170,25 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) ( nonce, nonceErr := hex.DecodeString(ask.Nonce) switch { case check.Node == "" || ask.Node != check.Node: - return HandOverAsk{}, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node) + return HandOverAsk{}, nil, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node) case ask.Expires.IsZero() || !now.Before(ask.Expires): - return HandOverAsk{}, "the ask has expired" + return HandOverAsk{}, nil, "the ask has expired" case ask.Expires.After(now.Add(HandOverGood + HandOverSkew)): - return HandOverAsk{}, "the ask is good for longer than an ask may be" + return HandOverAsk{}, nil, "the ask is good for longer than an ask may be" case nonceErr != nil || len(nonce) < 16: - return HandOverAsk{}, "the ask carries no nonce of its own" + return HandOverAsk{}, nil, "the ask carries no nonce of its own" case ask.By == "": - return HandOverAsk{}, "the ask names nobody as asking" + return HandOverAsk{}, nil, "the ask names nobody as asking" case len(ask.By) > MaxHandOverBy || strings.ContainsAny(ask.By, "\r\n\x00"): - return HandOverAsk{}, "who asked is not one short line" + return HandOverAsk{}, nil, "who asked is not one short line" case withPath && ask.Path == "": - return HandOverAsk{}, "the ask names no directory" + return HandOverAsk{}, nil, "the ask names no directory" case !withPath && ask.Path != "": - return HandOverAsk{}, "the ask names a path, and this ask takes none" + return HandOverAsk{}, nil, "the ask names a path, and this ask takes none" case !nonces.take(ask.Nonce, ask.Expires, now): - return HandOverAsk{}, "the ask was taken already" + return HandOverAsk{}, nil, "the ask was taken already" } - return ask, "" + return ask, signed.Ask, "" } // HandOverReply is what a hand-over request is answered with, and whether to answer: a request with no reply @@ -320,3 +327,104 @@ func JudgeDirectoriesReply(check HandOverCheck, handle JudgeDirectoriesFunc, rep type DirectoryJudgements interface { HearDirectoryJudgements(JudgeDirectoriesFunc) } + +// The take act on an accessed folder, asked over the bus (novox/hq ADR 0325 §8, issue 567). +// +// A folder that already holds data is kept only on the operator's word: the access waits (§7), the controller asks +// the operator once for that folder on that machine (ADR 0259), and on the approved warrant asks this node's engine +// on AskAccessActSubject, signed as a hand-over is and held to the same checks (believed), under a signing context +// of its own, naming the act, the access, its folder as the engine states it, and the warrant. The engine judges +// every value against what it was last declared and what is on disk, answers at once that the act started or why +// it is refused, adds the users' entries to everything under the folder (following no link, entering no other +// filesystem), and states the act on the access's verdict (Act). Adding or dropping a user of a folder already +// taken asks nobody: the engine does it on the apply that declares it. + +// AskAccessActSubject is where the controller asks this node's engine to run an act on one access's folder. +func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" } + +// AccessActContext is prefixed to an act ask's bytes before signing: never another ask's, nor a declaration's. +const AccessActContext = "novox-mesh access-act v2\n" + +// The walks under an accessed folder. Only ActTake is asked; ActAdd and ActDrop are what an apply starts. +const ( + // ActTake adds the users' entries to the folder and everything under it, on a warrant (ADR 0325 §8). + ActTake = "take" + // ActAdd adds the entries of a user that joined a taken folder; ActDrop removes exactly a dropped user's. + ActAdd = "add" + ActDrop = "drop" +) + +// AccessActAsk is an act's own fields, beside a hand-over's (node, path, by, expires, nonce): which act, the +// access by its resource id, and the warrant the operator approved. +type AccessActAsk struct { + Act string `json:"act"` + Access string `json:"access"` + Warrant string `json:"warrant"` +} + +// AccessAct is one act asked and believed, as the handler is given it. +type AccessAct struct { + Act, Access, Path, Warrant, By string +} + +// MaxActField bounds an access id and a warrant id. +const MaxActField = 200 + +// AccessActFunc judges and runs one act: the words to answer with, or the refusal. +type AccessActFunc func(AccessAct) (string, error) + +func refusedAct(why string) HandOverAnswer { + return HandOverAnswer{Refused: why + "; nothing was changed"} +} + +// AnswerAccessAct is the engine's judgement of one act ask's bytes: believed as a hand-over is, then the act, the +// access and the warrant — **nothing is done without a warrant** — and only then the handler. +func AnswerAccessAct(check HandOverCheck, handle AccessActFunc, body []byte) HandOverAnswer { + if handle == nil { + return refusedAct("this engine runs no act on an accessed folder") + } + ask, signed, refused := believedBytes(check, AccessActContext, true, body) + if refused != "" { + return refusedAct(refused) + } + var act AccessActAsk + if err := json.Unmarshal(signed, &act); err != nil { + return refusedAct("the signed ask is not an act's ask: " + err.Error()) + } + act.Access, act.Warrant = strings.TrimSpace(act.Access), strings.TrimSpace(act.Warrant) + switch { + case act.Warrant == "": + return refusedAct("the ask names no warrant, and an act on an accessed folder runs only on one") + case act.Access == "": + return refusedAct("the ask names no access") + case act.Act != ActTake: + return refusedAct(fmt.Sprintf("the ask names the act %q, and the one act asked is %q", act.Act, ActTake)) + } + for _, v := range []string{act.Access, act.Warrant} { + if len(v) > MaxActField || strings.ContainsAny(v, "\r\n\x00/") { + return refusedAct("the ask's access or warrant is not one short name") + } + } + said, err := handle(AccessAct{Act: act.Act, Access: act.Access, Path: ask.Path, Warrant: act.Warrant, By: ask.By}) + if err != nil { + return refusedAct(err.Error()) + } + return HandOverAnswer{Said: said} +} + +// AccessActReply is what an act request is answered with, and whether to answer, as HandOverReply. +func AccessActReply(check HandOverCheck, handle AccessActFunc, reply string, body []byte) ([]byte, bool) { + if reply == "" { + return nil, false + } + answer, err := json.Marshal(AnswerAccessAct(check, handle, body)) + if err != nil { + return nil, false + } + return answer, true +} + +// AccessActs is a link that hears the controller asking this node to run an act on an accessed folder. Optional. +type AccessActs interface { + HearAccessActs(AccessActFunc) +} diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index fd27d0f..d4bebc4 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -66,7 +66,9 @@ type natsLink struct { searching *nats.Subscription // judging hears the controller asking for a read-only judgement of this node's directories (issue 358). judging *nats.Subscription - node string + // acting hears the controller asking for the take act on an access's folder (novox/hq ADR 0325 §8). + acting *nats.Subscription + node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). signer ed25519.PublicKey arrived chan Declaration @@ -259,6 +261,24 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) { } } +// HearAccessActs listens for the controller asking this node to run the take act on an access's folder (novox/hq ADR +// 0325 §8), on this machine's own subject, verified as a hand-over is (AnswerAccessAct). A bus whose user list is older +// than the grant refuses the subscription, said by the client; the controller then hears no answer. +func (l *natsLink) HearAccessActs(handle AccessActFunc) { + if handle == nil { + return + } + check := HandOverCheck{Signer: l.signer, Node: l.node} + acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) { + if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer { + _ = msg.Respond(body) + } + }) + if err == nil { + l.acting = acting + } +} + func (l *natsLink) Declarations() <-chan Declaration { return l.arrived } func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } func (l *natsLink) Lost() <-chan error { return l.lost } @@ -276,6 +296,9 @@ func (l *natsLink) Close() { if l.searching != nil { _ = l.searching.Unsubscribe() } + if l.acting != nil { + _ = l.acting.Unsubscribe() + } if l.judging != nil { _ = l.judging.Unsubscribe() } diff --git a/internal/link/messages.go b/internal/link/messages.go index 3f84232..ab77a04 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -258,6 +258,14 @@ const RootContract = 3 // know. const GrantContract = 4 +// IdentityContract is the statement of a host that reads a declaration's `identities`, a container's `identity` and +// an access's `users` and `taken` (novox/hq ADR 0325): it makes each module's account, runs a container as its +// module's identity, keeps each shared folder usable through ACL entries, states an access it may not keep yet as +// waiting with a wait of kind access, and hears the take act. It no longer grants an access's group (ADR 0312 is +// superseded): a `grant` still sent is accepted and not acted on. Like the fields before it, its presence is what +// tells the controller this host may be sent those fields; an older host refuses the whole declaration for them. +const IdentityContract = 5 + // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. @@ -296,11 +304,16 @@ const KindAccount = "account" // starting ReasonUsedAsFound, until the operator hands it to the mesh (`nox node hand-over ` on the control-node, novox/hq issue 356). const KindDirectory = "directory" -// KindAccess is a read-write access that grants its path's group (novox/hq ADR 0312): healthy while the group may -// write the path, unhealthy while it cannot — the operator's to decide, since the mesh changes nothing on the path -// (§2) — or while the path is owned by a group that is never granted (§4). +// KindAccess is an access whose folder the engine keeps for its users (novox/hq ADR 0325): healthy while every user +// has its effective right on the folder and the sweep's sample under it (§10), waiting while the mesh may not keep +// it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12). const KindAccess = "access" +// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet +// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at +// a folder. Unhealthy, said under the access that last named it, until the drop walk has run. +const KindKeptEntries = "kept-entries" + // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" @@ -413,6 +426,78 @@ type ResourceHealth struct { // Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one // setting of the module by name, the part, and what the operator gives, in words. Absent otherwise. Waits []health.Wait `json:"waits,omitempty"` + // AccessWaits is what an access stated waiting waits for (novox/hq ADR 0325 §7). Beside Waits rather than in + // it because mesh-sdk's health.Wait names a secret or a setting and nothing else; it moves there once that type + // has a kind. Absent otherwise. + AccessWaits []AccessWait `json:"access-waits,omitempty"` + // Act is the last walk under an access's folder (ADR 0325 §8): a take on a warrant, or an add or a drop of a + // user's entries. Absent on every other verdict, and on a folder never walked. + Act *AccessActRecord `json:"act,omitempty"` + // Entries is the listing of a kept folder's users, for an access's verdict (ADR 0325 §4): each identity, its + // use, and a note beside the tool runner's entry. Absent on a folder not kept. + Entries []KeptEntry `json:"entries,omitempty"` +} + +// WaitKindAccess is the kind of a wait on an access (novox/hq ADR 0325 §7). +const WaitKindAccess = "access" + +// AccessWait is one access waiting for the operator: the access by its module's own id for it, the part (its +// module), and what the operator gives, in words with no path (ADR 0234 §6): "let the mesh keep the shared folder +// movies", or "mount the disk that holds books". +type AccessWait struct { + // Kind is WaitKindAccess. + Kind string `json:"kind"` + Access string `json:"access"` + Part string `json:"part"` + What string `json:"what"` + // Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant + // given for another filesystem (the folder moved) does not answer it, and the controller asks again. + Placed string `json:"placed,omitempty"` +} + +// KeptEntry is one user of a kept folder as its listing says it. +type KeptEntry struct { + Identity string `json:"identity"` + UID int `json:"uid"` + Use string `json:"use"` + From string `json:"from"` + // Note is said beside the tool runner's entry (ADR 0325 §4): it stands for every module whose tools use the + // folder, at the widest use any of them declares. + Note string `json:"note,omitempty"` +} + +// The states of a walk under an accessed folder. +const ( + ActRunning = "running" + ActDone = "done" + // ActStopped is a walk that met its bound in entries or time: what it changed stays, and what it did not + // reach is not changed. Asked again, or on the next apply, it goes on. + ActStopped = "stopped" + ActFailed = "failed" +) + +// AccessActRecord is one walk under an access's folder, as the engine records it (novox/hq ADR 0325 §8). Counts +// only: never a path. +type AccessActRecord struct { + // Act is ActTake, ActAdd or ActDrop. + Act string `json:"act"` + Access string `json:"access"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + State string `json:"state"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended,omitzero"` + // Folders and Files are how many it changed; Kept how many already had their entries; Links the links it + // passed by unfollowed; Other the entries it left that are neither a file nor a folder; Mounts the folders on + // another filesystem it did not enter. + Folders int `json:"folders,omitempty"` + Files int `json:"files,omitempty"` + Kept int `json:"kept,omitempty"` + Links int `json:"links,omitempty"` + Other int `json:"other,omitempty"` + Mounts int `json:"mounts,omitempty"` + // Why is what stopped or failed it, in words with no path. + Why string `json:"why,omitempty"` } // SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times diff --git a/internal/link/queue.go b/internal/link/queue.go index 227dc1d..f24cdb1 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -62,8 +62,11 @@ type Queue struct { // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the // link (novox/hq issue 358). Nil answers every such ask with a refusal. JudgeDirectories JudgeDirectoriesFunc - Say Announce - Timeout time.Duration + // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved + // warrant (novox/hq ADR 0325 §8). Nil answers every such ask with a refusal. + AccessAct AccessActFunc + Say Announce + Timeout time.Duration once sync.Once wake chan struct{} diff --git a/internal/link/run.go b/internal/link/run.go index b61782e..f45d808 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -252,6 +252,10 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou if j, ok := link.(DirectoryJudgements); ok { j.HearDirectoryJudgements(queue.JudgeDirectories) } + // And asking for the take act on an access's folder, on a warrant (novox/hq ADR 0325 §8). + if s, ok := link.(AccessActs); ok { + s.HearAccessActs(queue.AccessAct) + } for { // Asked to stop — by standing aside, say — nothing further is taken in. diff --git a/internal/store/kept.go b/internal/store/kept.go new file mode 100644 index 0000000..4e9252e --- /dev/null +++ b/internal/store/kept.go @@ -0,0 +1,212 @@ +package store + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" +) + +// KeptName is the file, beside the node's state, of every shared folder the engine keeps (novox/hq ADR 0325 §5): +// whose entries it added, on which filesystem it was placed, whether it was taken and under which warrant, and how +// far its sweep got. Its own file for the reason the hand-overs have theirs: the take act, the watch and the sweep +// run beside an apply, whose save of the state would lose a record written into it meanwhile. +const KeptName = "kept-folders.json" + +// KeptPath is that file beside the state at statePath. +func KeptPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), KeptName) +} + +// KeptUser is one identity the mesh keeps an entry for on a folder: its name, its number, its use and where it +// runs (a container, or the tool runner standing for every module whose tools use the folder, ADR 0325 §4). +type KeptUser struct { + Identity string `json:"identity"` + UID uint32 `json:"uid"` + Use string `json:"use"` + From string `json:"from"` +} + +// How a folder came to be kept. +const ( + // KeptByWarrant is a folder that held data, taken on the operator's word (ADR 0325 §8). + KeptByWarrant = "warrant" + // KeptEmpty is a folder that held nothing yet when it was first kept (§5). + KeptEmpty = "empty" + // KeptMade is a folder the engine made under a parent on its placed filesystem (§9). + KeptMade = "made" +) + +// KeptFolder is one shared folder the engine keeps, or has seen and waits to keep. Counts and mount points only +// beside its own path: never a path under it. +type KeptFolder struct { + Path string `json:"path"` + // Mount is the mount point of the filesystem the folder was placed on when the engine first saw it there: + // later, a nearest existing parent on another one is the disk not there (ADR 0325 §9). + Mount string `json:"mount,omitempty"` + // Kept is that the mesh keeps it; How why (KeptByWarrant, KeptEmpty, KeptMade); Warrant and By the warrant and + // who asked, for a folder taken on one; At since when. + Kept bool `json:"kept,omitempty"` + How string `json:"how,omitempty"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at,omitzero"` + // Users are the identities whose entries are on everything under the folder; Want what the last apply + // declared. Where they differ a walk adds or removes exactly the difference. + Users []KeptUser `json:"users,omitempty"` + Want []KeptUser `json:"want,omitempty"` + // IDs are the accesses that last named the folder: what a leftover is said under once none does. + IDs []string `json:"ids,omitempty"` + // Placed names the filesystem the folder was kept on (the engine's PlacedID: its mount point, source and type), + // so another filesystem mounted at the same point is a move, as another mount point is. + Placed string `json:"placed,omitempty"` + // Fs is what Placed was made from beside the mount point: "uuid:", "fsid:" + // or "source::". Kept on this machine only, never said. Absent on a record from an + // earlier build, whose Placed is then not compared. + Fs string `json:"fs,omitempty"` + // TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the + // controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another + // asks again. One that names no filesystem this way (a mount point, from an unmerged build) is not taken. + TakenOn string `json:"taken_on,omitempty"` + // Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or + // one gone): taken out by a drop walk once that filesystem holds the folder again, said until then. + Left []KeptLeft `json:"left,omitempty"` + // Act is the last walk under the folder: a take, an add or a drop. + Act *KeptAct `json:"act,omitempty"` + // Sweep is how far the sweep got (ADR 0325 §10). + Sweep KeptSweep `json:"sweep"` + // Watches is how many folders under it are watched; OutOfWatches that the machine's watch limit was met, and + // the sweep alone finds what is made where no watch is (§6). + Watches int `json:"watches,omitempty"` + OutOfWatches bool `json:"out_of_watches,omitempty"` +} + +// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are. +type KeptLeft struct { + Mount string `json:"mount"` + // Placed names that filesystem as KeptFolder.Placed does: entries left on the filesystem the folder is on + // now are the folder's own, never walked away. + Placed string `json:"placed,omitempty"` + Fs string `json:"fs,omitempty"` + Users []KeptUser `json:"users"` + // Since is when they were left: a filesystem not back at its mount point within LeftFor is gone, and what was + // left on it is forgotten (said once). + Since time.Time `json:"since,omitzero"` +} + +// KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. +type KeptAct struct { + // Act is take, add or drop. + Act string `json:"act"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + State string `json:"state"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended,omitzero"` + // Folders and Files are the entries changed; Kept the ones that already had what they need; Links the links + // passed by unfollowed; Other the entries neither a file nor a folder; Mounts the folders on another + // filesystem not entered. + Folders int `json:"folders,omitempty"` + Files int `json:"files,omitempty"` + Kept int `json:"kept,omitempty"` + Links int `json:"links,omitempty"` + Other int `json:"other,omitempty"` + Mounts int `json:"mounts,omitempty"` + // Why is what stopped or failed it, in words with no path. + Why string `json:"why,omitempty"` +} + +// KeptSweep is the sweep's place in a kept folder: where the last look stopped, below the folder, by the names +// of each level (never said, only resumed from); when it last covered the folder whole; and what this pass found. +type KeptSweep struct { + Cursor []string `json:"cursor,omitempty"` + Covered time.Time `json:"covered,omitzero"` + // Read is how many entries this pass read so far; Narrowed how many gave a user less than its right and were + // put back; Unrepaired how many could not be (an owner's bits, which the mesh never changes, or a write that + // failed). + Read int `json:"read,omitempty"` + Narrowed int `json:"narrowed,omitempty"` + Unrepaired int `json:"unrepaired,omitempty"` + // LastUnrepaired is what the last whole pass could not put back: said until the next pass is through. + LastUnrepaired int `json:"last_unrepaired,omitempty"` + // Bound is that the last look ran out of its bound before the folder was covered. + Bound bool `json:"bound,omitempty"` +} + +// KeptFile is the kept folders' file, one writer at a time in this process. +type KeptFile struct { + path string + mu sync.Mutex +} + +// NewKeptFile is the kept folders' file beside the state at statePath. +func NewKeptFile(statePath string) *KeptFile { return &KeptFile{path: KeptPath(statePath)} } + +// All is every kept or seen folder, by path. None is an empty map; a file that cannot be read is an error. +func (k *KeptFile) All() (map[string]KeptFolder, error) { + k.mu.Lock() + defer k.mu.Unlock() + return k.load() +} + +// Update changes the record of one folder under the file's lock: change is given the record (empty when there is +// none) and whether there was one, and says whether to keep it; a record not kept is removed. +func (k *KeptFile) Update(path string, change func(f *KeptFolder, had bool) bool) error { + k.mu.Lock() + defer k.mu.Unlock() + all, err := k.load() + if err != nil { + return err + } + f, had := all[path] + f.Path = path + if change(&f, had) { + all[path] = f + } else { + delete(all, path) + } + return k.save(all) +} + +func (k *KeptFile) load() (map[string]KeptFolder, error) { + raw, err := os.ReadFile(k.path) + if errors.Is(err, fs.ErrNotExist) { + return map[string]KeptFolder{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading the kept folders: %w", err) + } + out := map[string]KeptFolder{} + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the kept folders (%s) are unreadable: %w", k.path, err) + } + return out, nil +} + +func (k *KeptFile) save(all map[string]KeptFolder) error { + raw, err := json.MarshalIndent(all, "", " ") + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(k.path), ".kept-folders-*.json") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(append(raw, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), k.path) +} diff --git a/internal/store/store.go b/internal/store/store.go index 44ad307..b1bba24 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -241,6 +241,23 @@ type State struct { // network stays declared: undeclaring brings nothing back (ADR 0118), and a private network // assigned again finds the tunnel's configuration retired rather than missing. Retired []Retired `json:"retired,omitempty"` + + // Identities is every module identity the engine made here (novox/hq ADR 0325 §3): its account and group, + // by name and number, and which of the two it made. An identity no longer declared is removed, and only + // what the engine made: an account or group that was there before is never the mesh's to remove. + Identities []MadeIdentity `json:"identities,omitempty"` +} + +// MadeIdentity is one module identity as the engine made it. +type MadeIdentity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` + // MadeUser and MadeGroup say which the engine made: only those are removed. + MadeUser bool `json:"made_user,omitempty"` + MadeGroup bool `json:"made_group,omitempty"` + At time.Time `json:"at"` } // Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq diff --git a/internal/system/identities.go b/internal/system/identities.go new file mode 100644 index 0000000..1c14265 --- /dev/null +++ b/internal/system/identities.go @@ -0,0 +1,87 @@ +package system + +import ( + "context" + "fmt" + "strconv" + "strings" +) + +// A module's identity as a system account (novox/hq ADR 0325 §3): an account and a group with exactly the numbers +// the controller allocated, no login shell, no home and no password. Optional beside System: a system that has +// none (android) makes no module identity, and the engine says so. + +// CreateSystemGroup makes a system group with gid. +func (arch) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error { + if _, err := run(ctx, "groupadd", "--system", "--gid", strconv.Itoa(gid), name); err != nil { + return fmt.Errorf("cannot create the group %q: %w", name, err) + } + return nil +} + +// CreateSystemAccount makes a system account with uid and the group gid as its own: no home made, `/` as the +// home field, nologin as its shell, and the password useradd leaves locked. +func (arch) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error { + if _, err := run(ctx, "useradd", "--system", "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(gid), + "--no-create-home", "--home-dir", "/", "--shell", "/usr/bin/nologin", name); err != nil { + return fmt.Errorf("cannot create the account %q: %w", name, err) + } + return nil +} + +// RemoveAccount removes an account, leaving every file it owns where it is. +func (arch) RemoveAccount(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "userdel", name); err != nil { + return fmt.Errorf("cannot remove the account %q: %w", name, err) + } + return nil +} + +// RemoveGroup removes a group. +func (arch) RemoveGroup(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "groupdel", name); err != nil { + return fmt.Errorf("cannot remove the group %q: %w", name, err) + } + return nil +} + +// CreateSystemGroup makes a system group with gid, with busybox's addgroup. +func (alpine) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error { + if _, err := run(ctx, "addgroup", "-S", "-g", strconv.Itoa(gid), name); err != nil { + return fmt.Errorf("cannot create the group %q: %w", name, err) + } + return nil +} + +// CreateSystemAccount makes a system account with busybox's adduser: -S system, -D no password, -H no home made. +func (alpine) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error { + group, err := run(ctx, "getent", "group", strconv.Itoa(gid)) + if err != nil { + return fmt.Errorf("cannot create the account %q: its group gid %d is not here: %w", name, gid, err) + } + groupName := name + if n, _, ok := strings.Cut(group, ":"); ok { + groupName = n + } + if _, err := run(ctx, "adduser", "-S", "-D", "-H", "-h", "/", "-s", "/sbin/nologin", "-u", strconv.Itoa(uid), + "-G", groupName, name); err != nil { + return fmt.Errorf("cannot create the account %q: %w", name, err) + } + return nil +} + +// RemoveAccount removes an account with busybox's deluser. +func (alpine) RemoveAccount(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "deluser", name); err != nil { + return fmt.Errorf("cannot remove the account %q: %w", name, err) + } + return nil +} + +// RemoveGroup removes a group with busybox's delgroup. +func (alpine) RemoveGroup(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "delgroup", name); err != nil { + return fmt.Errorf("cannot remove the group %q: %w", name, err) + } + return nil +}