From 51e97a8c14058fd58e108a045b5d94744f23a6ee Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:43:04 +0200 Subject: [PATCH 01/10] Give every module an identity and keep shared folders through ACL entries (hq ADR 0325, issue 567) A module's containers ran as whatever their image chose and shared folders were made writable by putting accounts in the folder's group (ADR 0312), which ADR 0325 supersedes. The engine now makes each declared module identity as a system account with the controller's numbers, runs a container declaring `identity` as it, and keeps each placed shared folder usable through one POSIX ACL entry per user, set through the kernel's xattrs on descriptors it opened itself. A folder holding data is kept only after the operator's warrant (the take act); a missing folder is made where its data lives and waits where its disk is not; root's data is never kept. A watch and a bounded sweep put back what an app narrows. The group grant is removed, and what it put an account in is taken back. Reuses the descriptor walks, the create act, the missing-path wait and the warrant- backed act message from the paused issue 567 branch. --- cmd/mesh-host/grants.go | 181 -------- cmd/mesh-host/grants_test.go | 138 ------ cmd/mesh-host/kept.go | 408 ++++++++++++++++++ cmd/mesh-host/kept_test.go | 47 ++ cmd/mesh-host/main.go | 35 +- cmd/mesh-host/units_test.go | 6 +- internal/apply/access_test.go | 23 - internal/apply/acl.go | 227 ++++++++++ internal/apply/apply.go | 194 +++++---- internal/apply/grants.go | 514 ---------------------- internal/apply/grants_test.go | 559 ------------------------ internal/apply/identities.go | 253 +++++++++++ internal/apply/identities_test.go | 251 +++++++++++ internal/apply/keep.go | 579 +++++++++++++++++++++++++ internal/apply/keeper_linux.go | 319 ++++++++++++++ internal/apply/kept_test.go | 589 ++++++++++++++++++++++++++ internal/apply/mesh_names_test.go | 6 +- internal/apply/placed.go | 195 +++++++++ internal/apply/plan.go | 2 +- internal/apply/reads_test.go | 2 +- internal/apply/roots.go | 147 +++++++ internal/apply/schedule.go | 17 +- internal/apply/vocabulary_test.go | 6 +- internal/apply/walk.go | 67 +++ internal/apply/walk_linux.go | 536 +++++++++++++++++++++++ internal/apply/walk_other.go | 55 +++ internal/apply/watch_linux.go | 292 +++++++++++++ internal/declaration/declaration.go | 260 +++++++++--- internal/declaration/identity_test.go | 61 +++ internal/link/access_act_test.go | 68 +++ internal/link/handover.go | 136 +++++- internal/link/hearing_nats.go | 25 +- internal/link/messages.go | 83 +++- internal/link/queue.go | 7 +- internal/link/run.go | 4 + internal/store/kept.go | 183 ++++++++ internal/store/store.go | 17 + internal/system/identities.go | 87 ++++ 38 files changed, 4964 insertions(+), 1615 deletions(-) delete mode 100644 cmd/mesh-host/grants.go delete mode 100644 cmd/mesh-host/grants_test.go create mode 100644 cmd/mesh-host/kept.go create mode 100644 cmd/mesh-host/kept_test.go create mode 100644 internal/apply/acl.go delete mode 100644 internal/apply/grants.go delete mode 100644 internal/apply/grants_test.go create mode 100644 internal/apply/identities.go create mode 100644 internal/apply/identities_test.go create mode 100644 internal/apply/keep.go create mode 100644 internal/apply/keeper_linux.go create mode 100644 internal/apply/kept_test.go create mode 100644 internal/apply/placed.go create mode 100644 internal/apply/roots.go create mode 100644 internal/apply/walk.go create mode 100644 internal/apply/walk_linux.go create mode 100644 internal/apply/walk_other.go create mode 100644 internal/apply/watch_linux.go create mode 100644 internal/declaration/identity_test.go create mode 100644 internal/link/access_act_test.go create mode 100644 internal/store/kept.go create mode 100644 internal/system/identities.go diff --git a/cmd/mesh-host/grants.go b/cmd/mesh-host/grants.go deleted file mode 100644 index 892ca57..0000000 --- a/cmd/mesh-host/grants.go +++ /dev/null @@ -1,181 +0,0 @@ -package main - -import ( - "sort" - "strings" - "sync" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/link" -) - -// A read-write access whose group cannot write is never quiet (novox/hq ADR 0312 §2). -// -// The engine grants a read-write access's path group to the identities the module's code runs as, and changes -// nothing on the path: whether its group may write is the operator's, as the data is. So an access whose path's -// group has no write permission is stated, on every look, as its module's resource of kind `access`, unhealthy, -// saying so and that the operator decides — and one owned by a group that is never granted (§4) is stated -// unhealthy saying that. The controller raises it as that module's condition on two statements in a row and -// clears it on the first healthy one, as it does every resource's health (ADR 0240). The path is the target, -// never in the reason: the controller says a reason on the operator's channel, which withholds a path (ADR 0234 -// §6). - -// grantedAccess is one read-write access that grants, as the judge reads it. -type grantedAccess struct { - module, id, path string -} - -// accessJudge reads, on every look, whether each read-write access that grants may write its path through its -// group. Safe for the apply and the looking loop at once. -type accessJudge struct { - mu sync.Mutex - accesses []grantedAccess - // was is each access's last verdict, with since when and how many looks in a row found it unhealthy. - was map[string]link.ResourceHealth - - // reading guards inFlight: every path whose read has not returned yet. - reading sync.Mutex - inFlight map[string]bool -} - -// accesses is the one access judge on this machine. -var accesses = newAccessJudge() - -func newAccessJudge() *accessJudge { - return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}} -} - -// set is what the declaration just applied asks: every read-write access of a module that names a grant, not -// held as found. An access no longer declared is forgotten. -func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) { - var as []grantedAccess - if d != nil { - for _, r := range d.Resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite || held[a.ID] { - continue - } - at := strings.LastIndex(a.ID, ".") - if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { - continue - } - as = append(as, grantedAccess{module: a.ID[:at], id: a.ID, path: a.Path}) - } - } - sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id }) - j.mu.Lock() - defer j.mu.Unlock() - j.accesses = as - declared := map[string]bool{} - for _, a := range as { - declared[a.id] = true - } - for id := range j.was { - if !declared[id] { - delete(j.was, id) - } - } -} - -// lookBound is how long a look waits for one accessed path to answer. A hard network mount whose server is down -// blocks a stat for ever, and one access must never stop every health statement this machine makes. -var lookBound = 5 * time.Second - -// pathGroupBounded reads a path's group within lookBound, outside any lock. A read that has not answered is said -// so; while it still hangs, later looks do not start another on the same path, so a dead mount costs one blocked -// read, not one per look. -func (j *accessJudge) pathGroupBounded(path string) (apply.PathGroup, bool, error) { - j.reading.Lock() - if j.inFlight[path] { - j.reading.Unlock() - return apply.PathGroup{}, false, nil - } - j.inFlight[path] = true - j.reading.Unlock() - type read struct { - g apply.PathGroup - err error - } - done := make(chan read, 1) - go func() { - g, err := apply.GroupOfPath(path) - j.reading.Lock() - delete(j.inFlight, path) - j.reading.Unlock() - done <- read{g, err} - }() - select { - case r := <-done: - return r.g, true, r.err - case <-time.After(lookBound): - return apply.PathGroup{}, false, nil - } -} - -// look reads every access now: healthy when its group may write and may be granted, unhealthy saying why -// otherwise. Whether any verdict changed since the last look, for the loop to say it. The paths are read with no -// lock held, each within lookBound. -func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { - j.mu.Lock() - as := append([]grantedAccess(nil), j.accesses...) - j.mu.Unlock() - - type verdict struct{ state, reason string } - read := make([]verdict, len(as)) - for i, a := range as { - state, reason := link.StateHealthy, "" - g, answered, err := j.pathGroupBounded(a.path) - switch { - case !answered: - state, reason = link.StateUnhealthy, "its path does not answer: reading which group owns it did not "+ - "return within "+lookBound.String() - case err != nil: - state, reason = link.StateUnhealthy, "its group cannot be read: "+strings.ReplaceAll(err.Error(), a.path, "the path") - case apply.GrantRefusal(g) != nil: - state, reason = link.StateUnhealthy, "its group is never granted: "+apply.GrantRefusal(g).Error() - case !g.Writable: - state, reason = link.StateUnhealthy, apply.CannotWriteWords(g) - } - read[i] = verdict{state, reason} - } - - j.mu.Lock() - defer j.mu.Unlock() - changed := false - out := make([]link.ResourceHealth, 0, len(as)) - for i, a := range as { - state, reason := read[i].state, read[i].reason - v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path, - State: state, Reason: reason, Since: now.UTC()} - if was, had := j.was[a.id]; had && was.State == state && was.Reason == reason { - v.Since = was.Since - v.Streak = was.Streak - } else { - changed = true - } - if state != link.StateHealthy { - v.Streak++ - } - j.was[a.id] = v - out = append(out, v) - } - return out, changed -} - -// withAccesses adds to a statement every read-write access that grants, as its module's resource of kind access. -func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health { - h.Resources = append(h.Resources, vs...) - return h -} - -// accessesHealthy says no access in vs is anything but healthy. -func accessesHealthy(vs []link.ResourceHealth) bool { - for _, v := range vs { - if v.State != link.StateHealthy { - return false - } - } - return true -} diff --git a/cmd/mesh-host/grants_test.go b/cmd/mesh-host/grants_test.go deleted file mode 100644 index 27f1227..0000000 --- a/cmd/mesh-host/grants_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package main - -import ( - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-host/internal/apply" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/link" -) - -// A read-write access whose path's group cannot write is stated unhealthy on every look, saying so and that the -// operator decides, and one whose group is never granted says that (novox/hq ADR 0312 §2, §4). The path is the -// target and never in the reason (ADR 0234 §6); a read access, granting nothing, is not stated at all. -func TestAnAccessWhoseGroupCannotWriteIsStatedUnhealthyWithTheOperatorsRemedy(t *testing.T) { - spool, library, shared, read := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() - groups := map[string]apply.PathGroup{ - spool: {GID: 1002, Name: "downloads", Writable: false}, - library: {GID: 1001, Name: "media", Writable: true}, - shared: {GID: 0, Name: "root", Writable: true}, - } - was := apply.GroupOfPath - apply.GroupOfPath = func(path string) (apply.PathGroup, error) { return groups[path], nil } - t.Cleanup(func() { apply.GroupOfPath = was }) - wasRoot := apply.RootGroupIDs - apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} } - t.Cleanup(func() { apply.RootGroupIDs = wasRoot }) - - d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ - {"id":"nzbget.downloads","type":"access","path":"` + spool + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"importer.shared","type":"access","path":"` + shared + `","mode":"read-write","grant":{"account":"operator"}}, - {"id":"player.music","type":"access","path":"` + read + `","mode":"read"}]}`)) - if err != nil { - t.Fatal(err) - } - j := newAccessJudge() - j.set(d, nil) - at := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) - vs, changed := j.look(at) - if !changed || len(vs) != 3 { - t.Fatalf("stated %d accesses (changed %v), want the three read-write ones: %+v", len(vs), changed, vs) - } - got := map[string]link.ResourceHealth{} - for _, v := range vs { - got[v.Resource] = v - if v.Kind != link.KindAccess || strings.Contains(v.Reason, v.Target) { - t.Errorf("%s is stated as %q with its path in the reason %q", v.Resource, v.Kind, v.Reason) - } - } - if v := got["nzbget.downloads"]; v.State != link.StateUnhealthy || v.Module != "nzbget" || v.Target != spool || - !strings.Contains(v.Reason, "cannot write") || !strings.Contains(v.Reason, "operator's to decide") || v.Streak != 1 { - t.Errorf("an access whose group cannot write is stated %+v", v) - } - if v := got["importer.shared"]; v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "never granted") { - t.Errorf("an access of the root group is stated %+v", v) - } - if v := got["importer.music"]; v.State != link.StateHealthy { - t.Errorf("an access whose group may write is stated %+v", v) - } - if h := withAccesses(&link.Health{}, vs); accessesHealthy(vs) || len(h.Resources) != 3 { - t.Errorf("the statement does not carry the accesses as unhealthy: %+v", h.Resources) - } - - // Looked at again: the same verdict, its since kept and its streak counted. - vs, changed = j.look(at.Add(time.Minute)) - for _, v := range vs { - if v.Resource == "nzbget.downloads" && (changed || v.Streak != 2 || !v.Since.Equal(at)) { - t.Errorf("the second look is %+v (changed %v)", v, changed) - } - } - // The operator gives the group write: healthy at the next look, and nothing the engine did made it so. - groups[spool] = apply.PathGroup{GID: 1002, Name: "downloads", Writable: true} - vs, _ = j.look(at.Add(2 * time.Minute)) - for _, v := range vs { - if v.Resource == "nzbget.downloads" && v.State != link.StateHealthy { - t.Errorf("an access whose group was given write is stated %+v", v) - } - } -} - -// A path that does not answer — a hard network mount whose server is down — is stated unhealthy within the bound, -// and stops neither this look nor the next: one blocked read per path, never one per look. -func TestAnAccessWhosePathDoesNotAnswerIsStatedWithinTheBound(t *testing.T) { - hung, library := t.TempDir(), t.TempDir() - release, finished := make(chan struct{}), make(chan struct{}) - var started atomic.Int32 - was, wasBound := apply.GroupOfPath, lookBound - apply.GroupOfPath = func(path string) (apply.PathGroup, error) { - if path == hung { - started.Add(1) - <-release - defer close(finished) - } - return apply.PathGroup{GID: 1001, Name: "media", Writable: true}, nil - } - lookBound = 50 * time.Millisecond - wasRoot := apply.RootGroupIDs - apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} } - t.Cleanup(func() { apply.GroupOfPath, lookBound, apply.RootGroupIDs = was, wasBound, wasRoot }) - // Run first: the read that hangs is let go and has returned before anything it read is put back. - t.Cleanup(func() { close(release); <-finished }) - - d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ - {"id":"nzbget.downloads","type":"access","path":"` + hung + `","mode":"read-write","grant":{}}, - {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{}}]}`)) - if err != nil { - t.Fatal(err) - } - j := newAccessJudge() - j.set(d, nil) - for look := 0; look < 3; look++ { - done := make(chan []link.ResourceHealth, 1) - go func() { vs, _ := j.look(time.Now()); done <- vs }() - select { - case vs := <-done: - for _, v := range vs { - switch v.Resource { - case "nzbget.downloads": - if v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "does not answer") { - t.Errorf("look %d: a path that does not answer is stated %+v", look, v) - } - case "importer.music": - if v.State != link.StateHealthy { - t.Errorf("look %d: the path that answers is stated %+v", look, v) - } - } - } - case <-time.After(5 * time.Second): - t.Fatalf("look %d did not return while one path hangs", look) - } - } - if n := started.Load(); n != 1 { - t.Errorf("%d reads were started on the path that hangs; want one", n) - } -} diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go new file mode 100644 index 0000000..0a4e36e --- /dev/null +++ b/cmd/mesh-host/kept.go @@ -0,0 +1,408 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// An access is judged by what each of its users may do (novox/hq ADR 0325 §7, §10, §12). +// +// On every look the engine judges each module's access as its module's resource of kind `access`: waiting, with a +// wait of kind access, while the mesh may not keep its folder yet or its disk is not there; unhealthy for root's +// data on its placed filesystem, and for a kept folder that gives a user less than its right — read by entry and +// mask together, on the folder and the sweep's sample under it, which the sweep puts back where it can; healthy +// otherwise. The folder is the target, never in a reason nor in a wait's words: the controller says them on the +// operator's channel, which withholds a path (ADR 0234 §6). + +// judgedAccess is one access as the judge reads it. +type judgedAccess struct { + module, id, name, path string + taken bool + users []store.KeptUser +} + +// accessJudge reads every access on every look. Safe for the apply and the looking loop at once. +type accessJudge struct { + mu sync.Mutex + accesses []judgedAccess + // was is each access's last verdict, with since when and how many looks in a row found it unhealthy. + was map[string]link.ResourceHealth + + // reading guards inFlight: every folder whose look has not returned yet. + reading sync.Mutex + inFlight map[string]bool +} + +// accesses is the one access judge on this machine. +var accesses = newAccessJudge() + +func newAccessJudge() *accessJudge { + return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}} +} + +// set is what the declaration just applied asks: every access of a module, not held as found, with the users of +// its folder as every access of that folder gives them. An access no longer declared is forgotten. +func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) { + var as []judgedAccess + if d != nil { + users := map[string]map[uint32]store.KeptUser{} + taken := map[string]bool{} + for _, r := range d.Resources { + if a, ok := r.(*declaration.Access); ok { + path := filepath.Clean(a.Path) + if users[path] == nil { + users[path] = map[uint32]store.KeptUser{} + } + taken[path] = taken[path] || a.Taken + for _, u := range a.Users { + had, seen := users[path][uint32(u.UID)] + if !seen || (u.Use == declaration.AccessReadWrite && had.Use != declaration.AccessReadWrite) { + users[path][uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), + Use: u.Use, From: u.From} + } + } + } + } + for _, r := range d.Resources { + a, ok := r.(*declaration.Access) + if !ok || held[a.ID] || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { + continue + } + at := strings.LastIndex(a.ID, ".") + if at <= 0 { + continue + } + path := filepath.Clean(a.Path) + ja := judgedAccess{module: a.ID[:at], id: a.ID, name: a.ID[at+1:], path: path, taken: taken[path]} + for _, u := range users[path] { + ja.users = append(ja.users, u) + } + sort.Slice(ja.users, func(x, y int) bool { return ja.users[x].UID < ja.users[y].UID }) + as = append(as, ja) + } + } + sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id }) + j.mu.Lock() + defer j.mu.Unlock() + j.accesses = as + declared := map[string]bool{} + for _, a := range as { + declared[a.id] = true + } + for id := range j.was { + if !declared[id] { + delete(j.was, id) + } + } +} + +// lookBound is how long a look waits for one accessed folder to answer. A hard network mount whose server is down +// blocks a stat for ever, and one access must never stop every health statement this machine makes. +var lookBound = 5 * time.Second + +// verdict is one access's judgement in a look. +type verdict struct { + state, reason string + waits []link.AccessWait + act *link.AccessActRecord + entries []link.KeptEntry +} + +// judgeBounded judges one access within lookBound, outside any lock. A look that has not answered is said so; +// while it still hangs, later looks start no other on the same folder, so a dead mount costs one blocked read. +func (j *accessJudge) judgeBounded(a judgedAccess) (verdict, bool) { + j.reading.Lock() + if j.inFlight[a.path] { + j.reading.Unlock() + return verdict{}, false + } + j.inFlight[a.path] = true + j.reading.Unlock() + done := make(chan verdict, 1) + go func() { + v := judgeAccess(a) + j.reading.Lock() + delete(j.inFlight, a.path) + j.reading.Unlock() + done <- v + }() + select { + case v := <-done: + return v, true + case <-time.After(lookBound): + return verdict{}, false + } +} + +// judgeAccess is one access's judgement now (ADR 0325 §7, §10, §12), the sweep of its kept folder included. +func judgeAccess(a judgedAccess) verdict { + v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0) + switch v.Kind { + case apply.VerdictWaitKeep: + return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " + + "operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess, + Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}} + case apply.VerdictWaitDisk: + return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)", + waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, + What: apply.DiskWords(a.name)}}} + case apply.VerdictRoot: + return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)} + case apply.VerdictBroken: + return verdict{state: link.StateUnhealthy, reason: strings.ReplaceAll(v.Why, a.path, "the folder")} + case apply.VerdictMake: + return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " + + "its data lives (novox/hq ADR 0325 §9)"} + case apply.VerdictNobody: + return verdict{state: link.StateHealthy} + } + out := verdict{state: link.StateHealthy, entries: entriesOf(a.users)} + rec, _ := keptNow(a.path) + if act := rec.Act; act != nil { + out.act = &link.AccessActRecord{Act: act.Act, Access: a.id, Warrant: act.Warrant, By: act.By, State: act.State, + Started: act.Started, Ended: act.Ended, Folders: act.Folders, Files: act.Files, Kept: act.Kept, + Links: act.Links, Other: act.Other, Mounts: act.Mounts, Why: act.Why} + } + if !rec.Kept { + out.state, out.reason = link.StateStarting, "the controller says it was taken; the next apply keeps it" + return out + } + look := apply.LookKept(a.path, a.users, apply.SweepLimits) + var faults []string + if look.Failed != "" { + faults = append(faults, "the sweep could not look: "+strings.ReplaceAll(look.Failed, a.path, "the folder")) + } + if len(look.Short) > 0 { + faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+ + "apply") + } + if n := look.Sweep.Unrepaired + look.Sweep.LastUnrepaired; n > 0 { + faults = append(faults, fmt.Sprintf("%d entries under it give a user less than its right and cannot be "+ + "put back: an owner's own bits, which the mesh never changes, or an entry the filesystem refused", n)) + } + if len(faults) > 0 { + out.state, out.reason = link.StateUnhealthy, strings.Join(faults, "; ") + return out + } + switch { + case rec.Act != nil && rec.Act.State == link.ActRunning: + out.state, out.reason = link.StateStarting, "the take act is adding its users' entries under it" + case !sameUsersList(rec.Users, rec.Want): + out.state, out.reason = link.StateStarting, "the entries under it are being changed to its users now" + } + return out +} + +// entriesOf is a kept folder's listing (ADR 0325 §4): each user, and beside the tool runner's what it stands for. +func entriesOf(users []store.KeptUser) []link.KeptEntry { + var out []link.KeptEntry + for _, u := range users { + e := link.KeptEntry{Identity: u.Identity, UID: int(u.UID), Use: u.Use, From: u.From} + if u.From == declaration.FromToolRunner { + e.Note = "the tool runner's account stands for every module whose tools use this folder, at the widest " + + "use any of them declares: a module that declared only read may write through it (novox/hq ADR " + + "0325 §4, until issue 569)" + } + out = append(out, e) + } + return out +} + +func sameUsersList(a, b []store.KeptUser) bool { + if len(a) != len(b) { + return false + } + seen := map[uint32]string{} + for _, u := range a { + seen[u.UID] = u.Use + } + for _, u := range b { + if use, ok := seen[u.UID]; !ok || use != u.Use { + return false + } + } + return true +} + +// keptNow is the record of one kept folder, as the engine's file holds it. +func keptNow(path string) (store.KeptFolder, bool) { + all, err := apply.KeptFolders.All() + if err != nil { + return store.KeptFolder{}, false + } + f, had := all[path] + return f, had +} + +// look reads every access now. Whether any verdict changed since the last look, for the loop to say it. +func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { + j.mu.Lock() + as := append([]judgedAccess(nil), j.accesses...) + j.mu.Unlock() + + read := make([]verdict, len(as)) + for i, a := range as { + v, answered := j.judgeBounded(a) + if !answered { + v = verdict{state: link.StateUnhealthy, reason: "its folder does not answer: judging it did not return " + + "within " + lookBound.String()} + } + read[i] = v + } + + j.mu.Lock() + defer j.mu.Unlock() + changed := false + out := make([]link.ResourceHealth, 0, len(as)) + for i, a := range as { + r := read[i] + v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path, + State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries} + if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason { + v.Since = was.Since + v.Streak = was.Streak + } else { + changed = true + } + if r.state == link.StateUnhealthy { + v.Streak++ + } + j.was[a.id] = v + out = append(out, v) + } + return out, changed +} + +// withAccesses adds to a statement every access, as its module's resource of kind access. +func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health { + h.Resources = append(h.Resources, vs...) + return h +} + +// accessesHealthy says no access in vs is anything but healthy. +func accessesHealthy(vs []link.ResourceHealth) bool { + for _, v := range vs { + if v.State != link.StateHealthy { + return false + } + } + return true +} + +// keeper runs the walks under kept folders beside the applies, one at a time (ADR 0325 §8): poked after every +// apply and every look, it walks whatever folder's entries under it are not what was declared. +type keeper struct { + running, following atomic.Bool + say func(string) + watch *apply.Watcher +} + +// keeping is the one keeper on this machine. +var keeping = &keeper{say: func(string) {}} + +// poke starts a reconcile of the kept folders unless one is running, and has the watch follow what is kept. +func (k *keeper) poke(ctx context.Context) { + // The watch is set beside, never in the caller's way: watching a new folder's tree reads every folder of it. + if k.following.CompareAndSwap(false, true) { + go func() { + defer k.following.Store(false) + k.follow() + }() + } + if !k.running.CompareAndSwap(false, true) { + return + } + go func() { + defer k.running.Store(false) + for _, line := range apply.ReconcileKept(ctx) { + k.say(line) + } + }() +} + +// follow has the watch watch every kept folder with its users. +func (k *keeper) follow() { + if k.watch == nil { + return + } + all, err := apply.KeptFolders.All() + if err != nil { + return + } + folders := map[string][]store.KeptUser{} + for path, f := range all { + if f.Kept && len(f.Want) > 0 { + folders[path] = f.Want + } + } + k.watch.Set(folders) +} + +// startKeeping opens the watch and has the keeper say what it does. A machine without the watch is said so, and +// its sweep alone finds what is made. +func startKeeping(ctx context.Context, say func(string)) { + keeping.say = say + w, err := apply.NewWatcher() + if err != nil { + say(fmt.Sprintf("the kept folders are not watched (%v): the sweep alone finds what is made in them", err)) + } else { + keeping.watch = w + go w.Run(ctx) + } + keeping.poke(ctx) +} + +// accessActFrom is the take act as the controller asks it over the link (ADR 0325 §8): judged against the +// declaration this node keeps, and run beside the applies. +func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc { + return func(act link.AccessAct) (string, error) { + kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + if errors.Is(err, store.ErrNothingDeclared) { + return "", errors.New("this node keeps no declaration, so it is declared no access") + } + if err != nil { + return "", err + } + d, err := declaration.Parse(kept.Declaration) + if err != nil { + return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + _, said, err := apply.TakeFolder(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By, + apply.ExecRunner) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err) + return "", err + } + return said, nil + } +} + +// withWaiting is held with every id of waiting beside it: what liveness does not judge. +func withWaiting(held, waiting map[string]bool) map[string]bool { + if len(waiting) == 0 { + return held + } + out := map[string]bool{} + for id := range held { + out[id] = true + } + for id := range waiting { + out[id] = true + } + return out +} diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go new file mode 100644 index 0000000..3bf831e --- /dev/null +++ b/cmd/mesh-host/kept_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// Row 4: the listing of a kept folder says, beside the tool runner's entry, that it stands for every module whose +// tools use the folder at the widest use (novox/hq ADR 0325 §4). +func TestTheListingSaysWhatTheToolRunnersEntryStandsFor(t *testing.T) { + entries := entriesOf([]store.KeptUser{ + {Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: declaration.FromContainer}, + {Identity: "nox", UID: 61003, Use: "read-write", From: declaration.FromToolRunner}, + }) + if entries[0].Note != "" || !strings.Contains(entries[1].Note, "widest use") { + t.Fatalf("the listing: %+v", entries) + } +} + +// Row 7, the engine's part: a folder that holds data and was never taken is said waiting with one wait of kind +// access — the access by its module's own id, the part its module, and words with no path (novox/hq ADR 0325 §7). +func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { + dir := t.TempDir() + lib := filepath.Join(dir, "movies") + if err := os.MkdirAll(filepath.Join(lib, "a film"), 0o755); err != nil { + t.Fatal(err) + } + v := judgeAccess(judgedAccess{module: "radarr", id: "radarr.movies", name: "movies", path: lib, + users: []store.KeptUser{{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: "container"}}}) + if v.state != link.StateWaiting || len(v.waits) != 1 { + t.Fatalf("not waiting with one wait: %+v", v) + } + raw, _ := json.Marshal(v.waits[0]) + if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` { + t.Fatalf("the wait reads %s", raw) + } + if strings.Contains(v.reason, dir) { + t.Fatalf("the reason names the path: %s", v.reason) + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index feb1da7..a1d21ab 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -241,6 +241,8 @@ func run(ctx context.Context, command string, opts options) error { // found (novox/hq issue 339). handedOverFrom(opts.state) foundState.Store(opts.state) + // And what it knows of the shared folders it keeps is beside its state (novox/hq ADR 0325 §5). + apply.KeptFolders = store.NewKeptFile(opts.state) // And the guard asks the running container runtimes where they keep their data (novox/hq issue 339). apply.AskRuntimes = apply.ExecRunner switch command { @@ -1121,6 +1123,8 @@ func runLink(ctx context.Context, opts options) error { // this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224). sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state))) go sched.Run(ctx) + // The watch on every kept folder and the walks under them, beside the applies (novox/hq ADR 0325 §6, §8). + startKeeping(ctx, func(line string) { say(line) }) // The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every // report this host makes carries what it says. @@ -1232,8 +1236,10 @@ func runLink(ctx context.Context, opts options) error { // A read-only judgement of the directories this node is declared, asked by the controller (novox/hq // issue 358): which the walk that follows no link another account could make would refuse. JudgeDirectories: judgeDirectoriesFrom(opts.state), - Say: say, - Timeout: opts.timeout, + // The take act on an access's folder, on the operator's warrant (novox/hq ADR 0325 §8). + AccessAct: accessActFrom(opts.state, opts.timeout), + Say: say, + Timeout: opts.timeout, } worker := make(chan struct{}) go func() { @@ -1596,7 +1602,8 @@ const ( // judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 -// rule 6). +// rule 6) — but for the sweep of a kept folder, which ADR 0325 §10 has every look make: it puts back an entry or a +// mask an app narrowed, and nothing else. func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() @@ -1635,8 +1642,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - // Every read-write access that grants, read on every look (novox/hq ADR 0312 §2). + // Every access, judged on every look, its kept folder swept (novox/hq ADR 0325 §10); and a walk stopped + // at its bound goes on. accessSt, accessesChanged := accesses.look(time.Now()) + keeping.poke(ctx) owed = owed || accessesChanged if accessesChanged { for _, v := range accessSt { @@ -1691,10 +1700,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa // healthAsReported is a statement as the report and the event carry it. func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { - // GrantContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a - // user's declared `root` (novox/hq ADR 0266), and grants a read-write access's path group (novox/hq ADR 0312), - // which is what tells the controller it may be sent any of those fields. - h := &link.Health{Contract: link.GrantContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} + // IdentityContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a + // user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, a container's identity and + // an access's users (novox/hq ADR 0325), which is what tells the controller it may be sent any of those fields. + h := &link.Health{Contract: link.IdentityContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, @@ -2075,7 +2084,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just // applied, so every resource this apply started says `starting` and the gate waits for it. if j := judging; j != nil { - j.Set(liveness.LongRunning(declared, held)) + // A container that waits for an accessed folder not there was not created: the access says why, and + // liveness judges it neither alive nor dead (novox/hq ADR 0325 §9). + j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnAccess(declared)))) st, _ := j.Look(ctx) netJudge.declare(declaredResolvConf(declared)) var netSt *network.Statement @@ -2093,13 +2104,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // said with it, relogin needed included, in the report that says the apply. var accountSt *accounts.Statement if a := accountJudge; a != nil { - // A read-write access's account too (novox/hq ADR 0312 §6): relogin needed is said for a session of - // the account's own manager that began before the grant, as for a user resource's group. - a.Set(append(accounts.Of(declared, held), apply.GrantedAccounts(declared, held)...)) + a.Set(accounts.Of(declared, held)) as, _ := a.Look(ctx) accountSt = &as } accesses.set(declared, held) + // What the apply changed in who uses a kept folder is walked under it now, beside (ADR 0325 §8). + keeping.poke(ctx) accessSt, _ := accesses.look(time.Now()) report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), foundDirectories(updated)), accessSt) diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index f8cddc2..2036bce 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -87,8 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at}, }} h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as) - // At RootContract or later: an engine that also grants an access's group says GrantContract (novox/hq ADR 0312). - if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.GrantContract { + // At RootContract or later: an engine that also reads identities and keeps shared folders says IdentityContract + // (novox/hq ADR 0325), one above ADR 0312's GrantContract. + if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.IdentityContract || + link.IdentityContract != link.GrantContract+1 { t.Fatalf("contract %d", h.Contract) } got := map[string]string{} diff --git a/internal/apply/access_test.go b/internal/apply/access_test.go index f4f6187..f045bb5 100644 --- a/internal/apply/access_test.go +++ b/internal/apply/access_test.go @@ -3,7 +3,6 @@ package apply import ( "context" "os" - "strings" "testing" "github.com/novox/mesh-host/internal/store" @@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) { } } -// Absent is refused, not created. A bind mount whose source does not exist is made by the -// container runtime as root, with whatever mode it picks — the silent wrong-ownership -// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather -// than conjuring a directory it does not own. -func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) { - missing := t.TempDir() + "/media/library" // named, never created - d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`) - - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, - store.OriginDeclared, noServices, nil, nil) - if err == nil { - t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime") - } - if !strings.Contains(err.Error(), "the operator must provide") || - !strings.Contains(err.Error(), "does not own") { - t.Fatalf("the refusal does not say whose the path is: %v", err) - } - if _, statErr := os.Stat(missing); statErr == nil { - t.Fatal("the host created the path it does not own") - } -} - // Undeclaring an access never removes the path. Unassigning the module that reached the media // library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a // directory the mesh never made. The record is dropped; the operator's data is left exactly as it diff --git a/internal/apply/acl.go b/internal/apply/acl.go new file mode 100644 index 0000000..8874ec4 --- /dev/null +++ b/internal/apply/acl.go @@ -0,0 +1,227 @@ +package apply + +// The POSIX ACL a kept folder carries, as the kernel stores it (novox/hq ADR 0325 §5, issue 567). +// +// **Read and written as the kernel's own extended attribute, never through setfacl.** Linux keeps a file's access +// ACL in the attribute system.posix_acl_access and a folder's default ACL in system.posix_acl_default, each a +// little-endian header (version 2) followed by entries of a tag, a permission and an id. The engine reads and +// writes them on a descriptor it opened itself (fgetxattr, fsetxattr), so a name swapped for a link between the +// look and the change changes nothing, and no program is run per file of a library of millions. +// +// **What the mesh changes is narrow.** It adds or sets one named-user entry per uid it keeps, widens the mask to +// what the entries need, and on a file with no ACL yet writes the owning group's entry as exactly the group bits +// the mode had: owner, owning group, the owner's bits, others' bits and every other entry stay as found. + +import ( + "encoding/binary" + "errors" + "fmt" + "sort" +) + +// The attributes and the tags of the kernel's ACL format (linux/posix_acl_xattr.h). +const ( + aclAccessXattr = "system.posix_acl_access" + aclDefaultXattr = "system.posix_acl_default" + aclVersion = 2 + + aclUserObj = 0x01 + aclUser = 0x02 + aclGroupObj = 0x04 + aclGroup = 0x08 + aclMask = 0x10 + aclOther = 0x20 + + // aclUndefinedID is the id of an entry that names nobody: the owner, the owning group, the mask, others. + aclUndefinedID = 0xFFFFFFFF +) + +// The permission bits of one entry. +const ( + permRead = 4 + permWrite = 2 + permExec = 1 +) + +// aclEntry is one entry: its tag, its permission bits (rwx as 4, 2, 1) and, for a named user or group, its id. +type aclEntry struct { + Tag uint16 + Perm uint16 + ID uint32 +} + +// acl is a whole ACL as the kernel holds it. +type acl []aclEntry + +// decodeACL reads an attribute's bytes. An attribute that is not version 2, or not whole entries, is refused. +func decodeACL(raw []byte) (acl, error) { + if len(raw) < 4 || (len(raw)-4)%8 != 0 { + return nil, fmt.Errorf("an ACL attribute of %d bytes is not a header and whole entries", len(raw)) + } + if v := binary.LittleEndian.Uint32(raw); v != aclVersion { + return nil, fmt.Errorf("an ACL attribute of version %d, and the engine reads version %d", v, aclVersion) + } + out := make(acl, 0, (len(raw)-4)/8) + for at := 4; at < len(raw); at += 8 { + out = append(out, aclEntry{ + Tag: binary.LittleEndian.Uint16(raw[at:]), + Perm: binary.LittleEndian.Uint16(raw[at+2:]), + ID: binary.LittleEndian.Uint32(raw[at+4:]), + }) + } + return out, nil +} + +// encode is the attribute's bytes, the entries in the order the kernel requires: by tag, then by id. +func (a acl) encode() []byte { + sorted := append(acl(nil), a...) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].Tag != sorted[j].Tag { + return sorted[i].Tag < sorted[j].Tag + } + return sorted[i].ID < sorted[j].ID + }) + out := make([]byte, 4, 4+8*len(sorted)) + binary.LittleEndian.PutUint32(out, aclVersion) + for _, e := range sorted { + var b [8]byte + binary.LittleEndian.PutUint16(b[0:], e.Tag) + binary.LittleEndian.PutUint16(b[2:], e.Perm) + binary.LittleEndian.PutUint32(b[4:], e.ID) + out = append(out, b[:]...) + } + return out +} + +// minimalACL is the ACL a mode alone stands for: the owner's, the owning group's and others' bits. +func minimalACL(mode uint32) acl { + return acl{ + {Tag: aclUserObj, Perm: uint16(mode>>6) & 7, ID: aclUndefinedID}, + {Tag: aclGroupObj, Perm: uint16(mode>>3) & 7, ID: aclUndefinedID}, + {Tag: aclOther, Perm: uint16(mode) & 7, ID: aclUndefinedID}, + } +} + +// find is the index of the entry with tag and id, or -1. +func (a acl) find(tag uint16, id uint32) int { + for i, e := range a { + if e.Tag == tag && (e.ID == id || (tag != aclUser && tag != aclGroup)) { + return i + } + } + return -1 +} + +// perm is the bits of the entry with tag and id, and whether there is one. +func (a acl) perm(tag uint16, id uint32) (uint16, bool) { + if i := a.find(tag, id); i >= 0 { + return a[i].Perm, true + } + return 0, false +} + +// check refuses an ACL the kernel would: one owner's, owning group's and others' entry each, and a mask whenever +// a named entry is there. +func (a acl) check() error { + count := map[uint16]int{} + named := false + for _, e := range a { + count[e.Tag]++ + if e.Tag == aclUser || e.Tag == aclGroup { + named = true + } + } + if count[aclUserObj] != 1 || count[aclGroupObj] != 1 || count[aclOther] != 1 || count[aclMask] > 1 { + return errors.New("an ACL without exactly one owner's, owning group's and others' entry") + } + if named && count[aclMask] != 1 { + return errors.New("an ACL with named entries and no mask") + } + return nil +} + +// keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask +// widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps +// what it gave every other entry; with none yet (a file the mode alone described) the mask starts at the owning +// group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. +// Nothing else is changed. +func keptACL(a acl, need map[uint32]uint16) acl { + out := append(acl(nil), a...) + var widen uint16 + for uid, bits := range need { + widen |= bits + if i := out.find(aclUser, uid); i >= 0 { + out[i].Perm = bits + } else { + out = append(out, aclEntry{Tag: aclUser, Perm: bits, ID: uid}) + } + } + if i := out.find(aclMask, 0); i >= 0 { + out[i].Perm |= widen + return out + } + if len(need) == 0 { + return out + } + group, _ := out.perm(aclGroupObj, 0) + return append(out, aclEntry{Tag: aclMask, Perm: group | widen, ID: aclUndefinedID}) +} + +// withoutUsers is a with the named-user entry of each uid of drop taken out, and nothing else changed: the mask +// stays as it was, so no other entry loses a right. +func withoutUsers(a acl, drop map[uint32]bool) acl { + out := make(acl, 0, len(a)) + for _, e := range a { + if e.Tag == aclUser && drop[e.ID] { + continue + } + out = append(out, e) + } + return out +} + +// effective is the right a uid has through a, given the file's owner and mode: the owner's bits for the owner, +// else its named entry limited by the mask, else nothing a named entry gives (the owning group's and others' +// rights are not the mesh's to count on). An ACL with no mask limits nothing. +func effective(a acl, uid, owner uint32, mode uint32) uint16 { + if uid == owner { + return uint16(mode>>6) & 7 + } + bits, ok := a.perm(aclUser, uid) + if !ok { + return 0 + } + if mask, has := a.perm(aclMask, 0); has { + bits &= mask + } + return bits +} + +// needOn is what a use needs on a folder (read and enter; with read-write, write too) or on a file (read; with +// read-write, write): entering means nothing on a file, and an execute bit given to data would be a right nobody +// asked for. +func needOn(readWrite, folder bool) uint16 { + bits := uint16(permRead) + if readWrite { + bits |= permWrite + } + if folder { + bits |= permExec + } + return bits +} + +// permWords is bits as ls writes them: "rwx", "r-x". +func permWords(bits uint16) string { + out := []byte("---") + if bits&permRead != 0 { + out[0] = 'r' + } + if bits&permWrite != 0 { + out[1] = 'w' + } + if bits&permExec != 0 { + out[2] = 'x' + } + return string(out) +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1ff9a5b..c5f6028 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -262,6 +262,12 @@ func ApplyMindingWindows( } report = Report{} + // **No module identity is root** (novox/hq ADR 0325 §11): an identity or a container told a root number refuses + // the declaration whole, before anything of it is made or started. + if err := refuseRootIdentities(d); err != nil { + return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report} + } + declared := map[string]bool{} for _, r := range d.Resources { declared[r.Identity()] = true @@ -315,7 +321,7 @@ func ApplyMindingWindows( // (novox/hq ADR 0252), on the unit's rule above: two modules may each want the account in one group, // and the one going takes nothing the other still needs. wanted := groupsWanted(d.Resources) - // And what a record going hands to a resource that still asks for its group (novox/hq ADR 0312 §5). + // And what a record going hands to a resource that still asks for its group (novox/hq ADR 0252 §3). handed := Handed{} removeOrphan := func(orphan store.Applied) error { @@ -343,8 +349,16 @@ func ApplyMindingWindows( case declaration.TypeUser: action, detail, err = removeUser(ctx, sys, orphan, run, wanted, handed) case declaration.TypeAccess: - // The path is left as it is; what it granted an account is taken back (novox/hq ADR 0312 §5). - action, detail = removeAccess(ctx, sys, orphan, run, wanted, handed) + // The folder is left as it is: what the mesh added to it is taken out by the walk once no access names + // it (novox/hq ADR 0325 §8), never by removing the access's record. + action, detail = "forgotten", "an operator-owned folder is never the host's to remove; the entries "+ + "the mesh added go once no access names it" + if gave, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed); said != "" { + detail = joinDetail(detail, said) + if gave { + action = "restored" + } + } default: action, detail, err = remove(ctx, sys, orphan, run, made) } @@ -516,12 +530,10 @@ func ApplyMindingWindows( // which declared directories were already here: one this apply makes is the mesh's, one found may not be. refreshRuntimeRoots(ctx) in.refused = refusedPlaces(d.Resources) - // The group of every read-write access that grants, read once (novox/hq ADR 0312): what each container is - // created with, and what each account is asked to be in beside its user resources' groups — read before any - // orphan goes, so one access going leaves an account in a group another still grants. - in.grants = grantsOf(d.Resources, in.refused) - in.containerGroups = containerGroups(d.Resources, in.grants) - wanted.withGrants(d.Resources, in.grants) + // What every declared access asks of its folder, several accesses of one folder as one (novox/hq ADR 0325 §5), + // and every accessed folder, which no container mounts while it is not there (§9). + in.folders = foldersOf(d.Resources, in.refused) + in.accesses = accessPaths(d.Resources, in.refused) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} for _, resource := range d.Resources { @@ -563,6 +575,34 @@ func ApplyMindingWindows( sort.Strings(out) return out } + // **Every module's identity is made before anything of it is applied** (novox/hq ADR 0325 §3): a directory + // it owns is given to it, and a container is told to run as it. One that cannot be made — its name or numbers + // another's here — fails, naming whose, and its module is not attempted; the rest of the machine is. + var identityFailures []*Error + if len(d.Identities) > 0 { + tool, toolErr := toolFor(sys) + for _, id := range d.Identities { + var outcome Outcome + err := toolErr + if err == nil { + outcome, err = applyIdentity(ctx, tool, id, &known, run) + } + if err != nil { + // Counted with the others once the resources are through: here, before the guard, it would read + // as the guard not being up. + identityFailures = append(identityFailures, &Error{Resource: "identity " + id.Name, Err: err}) + log(fmt.Sprintf(" failed identity %s: %v", id.Name, err)) + gated[id.Module] = true + gatedWhy[id.Module] = "its identity " + id.Name + " could not be made, so nothing of this module " + + "was attempted" + continue + } + report.Outcomes = append(report.Outcomes, outcome) + if outcome.Action != "unchanged" { + log(fmt.Sprintf(" %s %s: %s", outcome.Action, outcome.ID, outcome.Detail)) + } + } + } for i, resource := range ordered { if !orphansRemoved && i == guardFirst { // **Only a guard that is up may let the filter go.** Removing the derived filter's @@ -925,6 +965,8 @@ func ApplyMindingWindows( } } + failures = append(failures, identityFailures...) + // A replacement that did not apply — failed, skipped behind its module's step, held — leaves // what it replaces running and recorded, said, for the next apply to hand over. unhanded := make([]string, 0, len(handover)) @@ -945,6 +987,19 @@ func ApplyMindingWindows( } } + // Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder. + if origin == store.OriginDeclared { + // A folder no access names any more has what the mesh added taken out by the next walk (ADR 0325 §8). + forgetUndeclared(in.folders) + // And an identity no longer declared goes, once what ran as it went above (ADR 0325 §3). + if tool, err := toolFor(sys); err == nil { + for _, o := range removeIdentities(ctx, tool, d.Identities, &known, run) { + report.Outcomes = append(report.Outcomes, o) + log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail)) + } + } + } + // A converged node whose found firewall was in force retires it only now, once everything — // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100), and on every // converged apply, not once (ADR 0168). Skipped, it is said: a step that does nothing is never @@ -1072,6 +1127,14 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru if err := refuseMounts(res, in.refused); err != nil { return begin(r), err } + // Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9). + if id := mountsMissing(res, in.accesses); id != "" { + out := begin(r) + out.Action = "waiting" + out.Detail = "not created: it mounts the access " + id + ", whose folder is not there; created once it " + + "is (novox/hq ADR 0325 §9)" + return out, nil + } } // Nothing below a home is touched through a link an account put there (novox/hq ADR 0266). // And nothing is unpacked or written into below an agent's home (agent_home_guard.go). @@ -1304,45 +1367,6 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O // library, a download spool that several modules share — and the host's only job is to be sure it // is there before anything mounts it. // -// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by -// the container runtime, as root, with whatever mode it picks — which is exactly the silent -// wrong-ownership 04-ISSUES/026 records. So the host checks first and says plainly that the -// operator must provide the path, rather than conjuring a directory it does not own and cannot -// give the right owner. Nothing is written, so this never reports a change: the machine did not -// move, the host merely confirmed a fact about it. -// -// **A read-write access grants its path's group** (novox/hq ADR 0312): read, never changed — grantAccess. -func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, - in inputs) (Outcome, error) { - out := begin(r) - info, err := os.Stat(r.Path) - if errors.Is(err, os.ErrNotExist) { - return out, fmt.Errorf( - "%s is not there, and the mesh does not own it — the operator must provide it. It is "+ - "shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+ - "nothing, so a missing one is said here rather than made as root by the container "+ - "runtime", r.Path) - } - if err != nil { - return out, err - } - if !info.IsDir() { - return out, fmt.Errorf( - "%s is not a directory, and an access is a shared directory the operator provides", - r.Path) - } - mode := declaration.AccessRead - if r.Mode != "" { - mode = r.Mode - } - out.Action = "unchanged" - out.Detail = "operator-owned; present, " + mode + ", nothing managed" - if err := grantAccess(ctx, sys, r, run, previous, in, &out); err != nil { - return out, err - } - return out, nil -} - // keepFound, when not nil, is where the original of a file this host has no record of is kept // before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { @@ -2160,10 +2184,10 @@ type inputs struct { // dirsBefore is every declared directory that was on the machine when this apply began (novox/hq issue // 339): one the apply itself made — a file's parent — is the mesh's, one that was there may not be. dirsBefore map[string]bool - // grants is what every read-write access that grants grants, by access id, read once (novox/hq ADR 0312); - // containerGroups the number of every group granted to a container, by container id. - grants map[string]grant - containerGroups map[string][]int + // folders is what every declared access asks of its folder, by path (novox/hq ADR 0325 §5); accesses every + // accessed folder with its access's id, which no container mounts while it is not there (§9). + folders map[string]*folderWant + accesses map[string]string // handed is every group a record that went this apply handed to a resource still asking for it. handed Handed } @@ -2240,15 +2264,13 @@ func (in inputs) reads(r *declaration.Container) map[string]string { // containerSpec is the identity of a declared container: everything that, if changed, means // the running container is no longer what was asked for. func containerSpec(r *declaration.Container, in inputs) string { - return containerSpecReading(r, in.declares, in.reads(r), in.containerGroups[r.ID]) + return containerSpecReading(r, in.declares, in.reads(r)) } // containerSpecReading is containerSpec with what the container reads already read — so an // applier that also records those digests reads each file once, and the label and the record // cannot disagree about a file that moved between two reads. -// -// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312). -func containerSpecReading(r *declaration.Container, declares, reads map[string]string, groups []int) string { +func containerSpecReading(r *declaration.Container, declares, reads map[string]string) string { keys := make([]string, 0, len(r.Env)) for k := range r.Env { keys = append(keys, k) @@ -2327,11 +2349,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s if h := r.Health; h.RunByRuntime() { b.WriteString("health " + strings.Join(runtimeCheckArgs(h), " ") + "\n") } - // And the groups its read-write accesses grant it (novox/hq ADR 0312 §6): a process's supplementary groups - // are fixed when the container is created, so one gained, dropped or renumbered is a container recreated - // with it. Only when there is one, so no container that is granted nothing has its digest moved. - for _, gid := range groups { - b.WriteString("group " + strconv.Itoa(gid) + "\n") + // And who it runs as (novox/hq ADR 0325 §2, §3): fixed when the container is created, so an identity told + // another way, or renumbered, is a container recreated with it. Only when there is one, so no container that + // runs as its image says has its digest moved. + if args := identityArgs(r); len(args) > 0 { + b.WriteString("identity " + strings.Join(args, " ") + "\n") } // **What this container reads is part of what it is.** // @@ -2433,8 +2455,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, out := begin(r) // Read once, so the spec and the record agree on what was read even if a file moves under them. reads := in.reads(r) - groups := in.containerGroups[r.ID] - want := containerSpecReading(r, in.declares, reads, groups) + want := containerSpecReading(r, in.declares, reads) out.reads = reads cri, err := containerRuntime(ctx, run) @@ -2462,7 +2483,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, } if r.RunOnce { - return applyRunOnce(ctx, r, run, cri, want, groups, previous) + return applyRunOnce(ctx, r, run, cri, want, previous) } before, err := containerState(ctx, r.Name, run) @@ -2507,7 +2528,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, // genuine recreate. The trade-off, stated: a container that was ALREADY stale when the host // upgraded — created against a file that has since changed — is not caught by this, and could // not be by the alternative either, which recreates it without knowing whether it needed to. - legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil, groups) && + legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil) && (wasReading == nil || sameReads(wasReading, reads)) // **A container a maintenance window holds still is left as it is** (novox/hq issue 224, ADR @@ -2577,7 +2598,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, for _, v := range r.Volumes { args = append(args, "--volume", v) } - args = append(args, groupAddArgs(groups)...) + args = append(args, identityArgs(r)...) // **The one check the container carries is the one the module declared** (ADR 0240 rule 3): an // exec command, or the image's own adopted by name, with the declared timing. Nothing else on the // machine sets a container's check. @@ -2634,9 +2655,9 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, } case len(reasons) > 0: out.Detail = "recreated to pick up " + strings.Join(reasons, ", ") - case len(groups) > 0 && before.Spec == containerSpecReading(r, in.declares, reads, nil): - // It was created before its read-write accesses granted it anything (novox/hq ADR 0312 §6). - out.Detail = "recreated to hold the groups its read-write accesses grant (gid " + gidWords(groups) + ")" + case r.RunAs != nil && before.Spec == specWithoutIdentity(r, in.declares, reads): + // It was created before it was told its module's identity (novox/hq ADR 0325 §3). + out.Detail = "recreated to run as its module's identity (novox/hq ADR 0325 §3)" default: out.Detail = "replaced; a container's configuration is fixed when it is created" } @@ -2676,7 +2697,7 @@ func runtimeCheckArgs(h *declaration.Health) []string { // - It writes a seed only if it is absent and never reconciles it, so what a running program // grows in that seed afterward is never wiped (04-ISSUES/035). The host's marker keeps the // step from re-running; the step's own code keeps it from clobbering on the pass it does run. -func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, groups []int, +func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, previous store.Applied) (Outcome, error) { out := begin(r) @@ -2696,7 +2717,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri // Run in the foreground so the runtime waits for the container and hands back its exit code. // No --detach and no --restart: a step that is restarted is not a step. - args := foregroundRunArgs(r, want, groups) + args := foregroundRunArgs(r, want) if _, err := run(ctx, cri, args...); err != nil { // A non-zero exit or a runtime that could not start it. Either way the step did not make @@ -2718,9 +2739,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri // exit code — no --detach, no --restart, because a step that is restarted is not a step. Shared by // a run-once step (novox/hq ADR 0052) and by one fire of a scheduled step (ADR 0053), which are the // same "run the container and let it exit" up to how often it happens. -// -// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312). -func foregroundRunArgs(r *declaration.Container, want string, groups []int) []string { +func foregroundRunArgs(r *declaration.Container, want string) []string { args := []string{"run", "--name", r.Name} for _, file := range r.EnvFile { args = append(args, "--env-file", file) @@ -2735,7 +2754,7 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st for _, v := range r.Volumes { args = append(args, "--volume", v) } - args = append(args, groupAddArgs(groups)...) + args = append(args, identityArgs(r)...) for _, h := range r.Hosts { args = append(args, "--add-host", h) } @@ -2744,6 +2763,31 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st return args } +// specWithoutIdentity is the spec r had before it was told who to run as. +func specWithoutIdentity(r *declaration.Container, declares, reads map[string]string) string { + plain := *r + plain.RunAs = nil + return containerSpecReading(&plain, declares, reads) +} + +// identityArgs is the runtime's arguments that run a container as its module's identity (novox/hq ADR 0325 §2): +// the image told by the variables it reads, or by the runtime's own user option, by number — a container's own +// account database is not the machine's. None for a container that runs as its image says. +func identityArgs(r *declaration.Container) []string { + id := r.RunAs + if id == nil { + return nil + } + if id.Mode == declaration.IdentityByRun { + return []string{"--user", strconv.Itoa(id.UID) + ":" + strconv.Itoa(id.GID)} + } + args := []string{"--env", id.UserEnv + "=" + strconv.Itoa(id.UID)} + if id.GroupEnv != "" { + args = append(args, "--env", id.GroupEnv+"="+strconv.Itoa(id.GID)) + } + return args +} + // ensureImage makes the pinned image present on the node without running anything. // // A service or a run-once container gets its image as a side effect of `docker run` — the first run diff --git a/internal/apply/grants.go b/internal/apply/grants.go deleted file mode 100644 index da702cf..0000000 --- a/internal/apply/grants.go +++ /dev/null @@ -1,514 +0,0 @@ -package apply - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/user" - "path/filepath" - "slices" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/novox/mesh-host/internal/accounts" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" - "github.com/novox/mesh-host/internal/system" -) - -// A read-write access grants the path's own group (novox/hq ADR 0312, issue 537). -// -// **An access is not owning.** ADR 0051 left the operator's shared paths exactly as they are: the engine -// confirms one is there and mounts it, and sets no owner, group or mode on it or below it. Nothing decided how -// a `read-write` access then becomes writable, and a module whose identity was not the path's owner could not -// write it at all — a download client could not remove a folder it had filed (issue 535). -// -// **The path's group is the operator's statement of who may share it.** So the engine reads, at apply, which -// group owns the accessed path — only the machine knows — and makes the identities the controller names for -// the access members of it: each container that mounts it gains the group as a supplementary group of its -// process (`--group-add`, by number, because a container's own group database is not the machine's), and the -// account that runs the module's tools is put in it as an account group (ADR 0252 §1). The module then writes -// the path as its group may, and as nothing else. -// -// **The path is never changed** (§2). A group that cannot write the path is the operator's choice, as the data -// is: the access grants nothing, is stated unhealthy on every look, saying so and that the operator decides, and -// nothing is done to make it writable. **A read access grants nothing** (§3). **The root group is never granted** (§4): a -// path owned by group 0, or by a group whose members act as root (accounts.RootGroups, the account judge's own -// closed list, judged by the numbers those groups have here), or by a number the machine names nothing for, is -// refused by name. -// -// **What was granted is taken back as ADR 0252 takes back** (§5): the access records each group it put the -// account in that the account was not in before; when no declared resource asks for it any more, the account -// leaves it, and a group the account held before is never recorded and never taken back. A container needs no -// record: the group is part of what it is created as, so one no longer granted is recreated without it. -// -// **And what runs the code is restarted to hold it** (§6). A container is recreated with the group, because -// its spec carries it. The tool runner — a process of the machine's own service manager with `User=` — is -// restarted because the controller names the access under its `restart-on`, and the access is reported -// updated exactly when the account's groups moved: that manager reads the account's groups from the database -// each time it starts the unit, so the restarted process holds the group with no new login. A unit of the -// account's own manager, or a login session, keeps the groups its login began with: the account judge -// (internal/accounts) says *relogin needed* there, for the access's account as for a user resource's. - -// PathGroup is the group that owns an accessed path, as the engine reads it. -type PathGroup struct { - GID int - // Name is the group's name in the machine's group database; empty when the database names nothing for - // the number. - Name string - // Writable is whether the group's write bit is set on the path itself. - Writable bool -} - -// Words names the group for a person: its name and number, or its number alone. -func (g PathGroup) Words() string { - if g.Name == "" { - return fmt.Sprintf("gid %d", g.GID) - } - return fmt.Sprintf("%s (gid %d)", g.Name, g.GID) -} - -// GroupOfPath reads which group owns a path, by its number, its name and whether it may write. Reads only. A -// variable so a test can stand in for a machine whose groups it cannot make. -var GroupOfPath = func(path string) (PathGroup, error) { - info, err := os.Stat(path) - if err != nil { - return PathGroup{}, err - } - _, gid, ok := statOwner(info) - if !ok { - return PathGroup{}, fmt.Errorf("the group that owns %s cannot be read on this machine", path) - } - g := PathGroup{GID: gid, Writable: info.Mode().Perm()&0o020 != 0} - if found, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil { - g.Name = found.Name - } - return g, nil -} - -// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine -// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine -// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number -// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and -// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can -// stand in. -var RootGroupIDs = func() map[int]string { - rootIDs.Lock() - defer rootIDs.Unlock() - if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute { - rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now() - } - return rootIDs.ids -} - -var rootIDs struct { - sync.Mutex - ids map[int]string - at time.Time -} - -func readRootGroupIDs() map[int]string { - out := map[int]string{} - if raw, err := os.ReadFile("/etc/group"); err == nil { - for name, gid := range groupEntries(string(raw)) { - if slices.Contains(accounts.RootGroups, name) { - out[gid] = name - } - } - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - for _, name := range accounts.RootGroups { - raw, err := exec.CommandContext(ctx, "getent", "group", name).Output() - if err != nil { - continue - } - for found, gid := range groupEntries(string(raw)) { - if found == name { - out[gid] = name - } - } - } - return out -} - -// groupEntries reads group database lines, name:password:gid:members, as name to number. -func groupEntries(raw string) map[string]int { - out := map[string]int{} - for _, line := range strings.Split(raw, "\n") { - fields := strings.Split(strings.TrimSpace(line), ":") - if len(fields) < 3 || strings.HasPrefix(fields[0], "#") { - continue - } - if gid, err := strconv.Atoi(fields[2]); err == nil { - out[fields[0]] = gid - } - } - return out -} - -// GrantRefusal is why a path's group is never granted (novox/hq ADR 0312 §4), or nil when it may be: group 0, a -// group whose members act as root by membership alone — judged by its NUMBER against every number those groups -// have here, and by its name — or a group this machine names nothing for, which cannot be judged at all. -func GrantRefusal(g PathGroup) error { - const never = "a shared path is never a way to root (novox/hq ADR 0312 §4); nothing was granted, and nothing " + - "was changed on it" - if g.GID == 0 { - return fmt.Errorf("it is owned by group 0 (%s), and a read-write access never grants the root group: %s", - orRoot(g.Name), never) - } - if root, is := RootGroupIDs()[g.GID]; is { - return fmt.Errorf("it is owned by gid %d, which is %s here, whose members act as root, and a read-write "+ - "access never grants such a group: %s", g.GID, root, never) - } - if slices.Contains(accounts.RootGroups, g.Name) { - return fmt.Errorf("it is owned by the group %s, whose members act as root, and a read-write access never "+ - "grants such a group: %s", g.Words(), never) - } - if g.Name == "" { - return fmt.Errorf("it is owned by gid %d, which this machine's group database names nothing, so whether "+ - "its members act as root cannot be judged, and an unjudged group is never granted: %s", g.GID, never) - } - return nil -} - -// CannotWriteWords is the reason an access whose group may not write its path is stated unhealthy with (novox/hq -// ADR 0312 §2). Never the path: the controller says a reason on the operator's channel, which withholds a path -// (ADR 0234 §6), and keeps the target as evidence. -func CannotWriteWords(g PathGroup) string { - return fmt.Sprintf("its group cannot write it: %s owns it without group write, and the mesh changes nothing on "+ - "an accessed path (novox/hq ADR 0312 §2) — whether its group may write is the operator's to decide", - g.Words()) -} - -func orRoot(name string) string { - if name == "" { - return "root" - } - return name -} - -// grant is what one read-write access grants, read once per apply. -type grant struct { - path string - group PathGroup - // refused is why nothing is granted: the path could not be read, or its group is never granted. - refused error -} - -// grants says the group is given: it may be granted, and it may write the path. A group that cannot write is -// granted nothing (novox/hq ADR 0312 §2) — the access is stated unhealthy for the operator to decide. -func (g grant) grants() bool { return g.refused == nil && g.group.Writable } - -// grantsOf reads the group of every read-write access that names a grant, by access id. One that is refused -// grants nothing — the access itself fails, saying why — and one whose path the engine refuses to place -// anything at (issue 339) is not read at all. -func grantsOf(resources []declaration.Resource, refused map[string]error) map[string]grant { - out := map[string]grant{} - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite { - continue - } - if why := refused[filepath.Clean(a.Path)]; why != nil { - out[a.ID] = grant{path: a.Path, refused: why} - continue - } - g, err := GroupOfPath(a.Path) - if err == nil { - err = GrantRefusal(g) - } - out[a.ID] = grant{path: a.Path, group: g, refused: err} - } - return out -} - -// containerGroups is, for each container, the number of every group a read-write access grants it, sorted and -// once each: what it is created with as supplementary groups. -func containerGroups(resources []declaration.Resource, grants map[string]grant) map[string][]int { - out := map[string][]int{} - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - for _, id := range a.Grant.Containers { - if !slices.Contains(out[id], g.group.GID) { - out[id] = append(out[id], g.group.GID) - } - } - } - for id := range out { - sort.Ints(out[id]) - } - return out -} - -// ContainerGroupsOf is containerGroups for a whole declaration, read now: what the scheduler arms a scheduled -// step with, so a fire runs with the groups an apply would have created it with. A path issue 339's guard refuses -// is not read, as the apply reads none. -func ContainerGroupsOf(d *declaration.Declaration) map[string][]int { - if d == nil { - return nil - } - return containerGroups(d.Resources, grantsOf(d.Resources, refusedPlaces(d.Resources))) -} - -// withGrants adds to what the declaration asks of each account the groups its read-write accesses grant (novox/hq -// ADR 0312 §5): one user resource or access going leaves an account in a group another still asks for. -func (w Wanted) withGrants(resources []declaration.Resource, grants map[string]grant) Wanted { - for _, r := range resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Grant.Account == "" { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - if w[a.Grant.Account] == nil { - w[a.Grant.Account] = map[string][]string{} - } - w[a.Grant.Account][g.group.Name] = append(w[a.Grant.Account][g.group.Name], a.ID) - } - return w -} - -// GrantedAccounts is every account a read-write access puts in its path's group, as the account judge reads -// them (novox/hq ADR 0312 §6): judged like a user resource's account groups, so a session of the account's own -// manager that began before is said *relogin needed*, and only there. Read now; a path whose group is refused -// or cannot be read asks nothing of the judge — the access says that itself. -func GrantedAccounts(d *declaration.Declaration, held map[string]bool) []accounts.Account { - if d == nil { - return nil - } - grants := grantsOf(d.Resources, refusedPlaces(d.Resources)) - var out []accounts.Account - for _, r := range d.Resources { - a, ok := r.(*declaration.Access) - if !ok || a.Grant == nil || a.Grant.Account == "" || held[a.ID] { - continue - } - g, granted := grants[a.ID] - if !granted || !g.grants() { - continue - } - at := strings.LastIndex(a.ID, ".") - if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { - continue - } - out = append(out, accounts.Account{Module: a.ID[:at], ID: a.ID, Name: a.Grant.Account, - Groups: []string{g.group.Name}}) - } - sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) - return out -} - -// grantAccess is applyAccess's grant (novox/hq ADR 0312): the account put in the path's group, what an earlier -// apply of this access put an account in and nobody asks for any more taken back, and the outcome said. The -// path itself is only read. -func grantAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, - in inputs, out *Outcome) error { - var g grant - granting := r.Grant != nil && r.Mode == declaration.AccessReadWrite - if granting { - var read bool - g, read = in.grants[r.ID] - if !read { - // An apply that read nothing ahead — a caller with no inputs — reads it here. - g = grantsOf([]declaration.Resource{r}, in.refused)[r.ID] - } - if g.refused != nil { - return fmt.Errorf("%s grants nothing: %w", r.Path, g.refused) - } - } - // A group that cannot write is granted nothing (§2): what an earlier apply granted is taken back below. - account := "" - if granting && g.group.Writable { - account = r.Grant.Account - } - if account != "" && g.group.Name == "" { - return fmt.Errorf("%s is owned by %s, which this machine's group database names nothing, and an account "+ - "is put in a group by name: %s was not (novox/hq ADR 0312 §1)", r.Path, g.group.Words(), account) - } - - var said []string - moved := false - // What this access put an account in before. An access whose account changed gives the old one back - // everything it put it in; the new one starts with nothing recorded. - recorded := previous.Groups - if previous.User != "" && previous.User != account { - gave, words := takeBack(ctx, sys, previous.User, r.ID, previous.Groups, nil, run, in.groups) - moved = moved || gave - said = append(said, words...) - recorded = nil - } - var kept []string - if account != "" { - groups, err := system.GroupsOf(ctx, system.Runner(run), account) - if err != nil { - return err - } - inGroup := setOf(groups)[g.group.Name] - if !inGroup { - if err := sys.AddUserToGroup(ctx, system.Runner(run), account, g.group.Name); err != nil { - return err - } - moved = true - said = append(said, fmt.Sprintf("put %s in %s, the group that owns it; the tool runner holds it once "+ - "restarted, a session that began before only after a new login", account, g.group.Name)) - } - // Recorded only when the mesh put it there, now or before, or a record that went handed it over: a group - // held before is never taken back. - if !inGroup || slices.Contains(recorded, g.group.Name) || in.handed.has(account, g.group.Name) { - kept = append(kept, g.group.Name) - } - var others []string - for _, x := range recorded { - if x != g.group.Name { - others = append(others, x) - } - } - gave, words := takeBack(ctx, sys, account, r.ID, others, &kept, run, in.groups) - moved = moved || gave - said = append(said, words...) - } - if len(kept) > 0 { - out.groups, out.user = kept, account - } - - if granting { - var to []string - if len(r.Grant.Containers) > 0 { - to = append(to, "its containers "+strings.Join(r.Grant.Containers, ", ")) - } - if account != "" { - to = append(to, account) - } - switch { - case !g.group.Writable: - out.Detail = joinDetail(out.Detail, fmt.Sprintf("its group %s cannot write it, so nothing is granted: "+ - "stated unhealthy, and nothing was changed on it — the operator decides", g.group.Words())) - case len(to) > 0: - out.Detail = joinDetail(out.Detail, fmt.Sprintf("grants %s to %s", g.group.Words(), strings.Join(to, " and "))) - } - } - if len(said) > 0 { - out.Detail = joinDetail(out.Detail, strings.Join(said, "; ")) - } - if moved { - // Updated exactly when the account's groups moved: what names this access under restart-on — the - // tool runner — is restarted to hold them (novox/hq ADR 0312 §6). - out.Action = "updated" - } - return nil -} - -// takeBack takes account out of every group of groups the mesh put it in for the resource id, unless another -// declared resource still asks for it — then it stays, and stays recorded on keep when keep is given, so it is -// taken back once nobody asks. A group it is no longer in is nothing to give back. Never fatal: a failure is -// said, and the group stays recorded where it can be. Whether any group was taken back, and what was said. -func takeBack(ctx context.Context, sys system.System, account, id string, groups []string, keep *[]string, - run Runner, wanted Wanted) (bool, []string) { - if len(groups) == 0 { - return false, nil - } - in, err := system.GroupsOf(ctx, system.Runner(run), account) - if err != nil { - if keep != nil { - *keep = append(*keep, groups...) - } - return false, []string{fmt.Sprintf("the groups the mesh put %s in (%s) were not given back: %v", account, - strings.Join(groups, ", "), err)} - } - already := setOf(in) - gave := false - var said []string - for _, g := range groups { - if !already[g] { - continue - } - if others := wanted.othersAsk(account, g, id); len(others) > 0 { - said = append(said, fmt.Sprintf("left %s in %s, which %s still asks for", account, g, strings.Join(others, ", "))) - if keep != nil { - *keep = append(*keep, g) - } - continue - } - gone, why := leaveGroup(ctx, sys, account, g, run) - if !gone && keep != nil { - *keep = append(*keep, g) - } - gave = gave || gone - said = append(said, account+" "+why) - } - return gave, said -} - -// removeAccess is an access no longer declared: the path left exactly as it is (novox/hq ADR 0051), and every -// group it put an account in taken back unless another declared resource still asks for it (ADR 0312 §5) — then -// handed to that resource, which records it as the mesh's and takes it back when it stops asking. -func removeAccess(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted, - handed Handed) (string, string) { - const left = "an operator-owned path is never the host's to remove" - if a.User == "" || len(a.Groups) == 0 { - return "forgotten", left - } - var stays []string - gave, said := takeBack(ctx, sys, a.User, a.ID, a.Groups, &stays, run, wanted) - for _, g := range stays { - if len(wanted.othersAsk(a.User, g, a.ID)) > 0 { - handed.hand(a.User, g) - } - } - action := "forgotten" - if gave { - action = "restored" - } - return action, joinDetail(left, strings.Join(said, "; ")) -} - -// groupAddArgs is the runtime's arguments that give a container's process each granted group as a supplementary -// group, by number: the container's own group database is not the machine's, and a name could mean another -// group there, or none. -func groupAddArgs(groups []int) []string { - var args []string - for _, gid := range groups { - args = append(args, "--group-add", strconv.Itoa(gid)) - } - return args -} - -func gidWords(groups []int) string { - words := make([]string, 0, len(groups)) - for _, gid := range groups { - words = append(words, strconv.Itoa(gid)) - } - return strings.Join(words, ", ") -} - -// Handed is every group a record that went in this apply put an account in, and another declared resource still -// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group -// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it -// there went first (novox/hq ADR 0252 §3, ADR 0312 §5). -type Handed map[string]map[string]bool - -func (h Handed) hand(account, group string) { - if h == nil { - return - } - if h[account] == nil { - h[account] = map[string]bool{} - } - h[account][group] = true -} - -func (h Handed) has(account, group string) bool { return h[account][group] } diff --git a/internal/apply/grants_test.go b/internal/apply/grants_test.go deleted file mode 100644 index e540a4d..0000000 --- a/internal/apply/grants_test.go +++ /dev/null @@ -1,559 +0,0 @@ -package apply - -import ( - "context" - "errors" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/accounts" - "github.com/novox/mesh-host/internal/declaration" - "github.com/novox/mesh-host/internal/store" -) - -// Defends novox/hq ADR 0312 and issue 537: a read-write access grants its path's own group to the containers and -// the account the controller names, and the path is never changed; a read access grants nothing; the root group, -// and a group whose members act as root, are refused by name; what was granted is taken back as ADR 0252 takes -// back; a container is recreated with the group, and a unit of the machine's own manager running as the account -// is restarted once the account holds it. - -// grantMachine is the group database of groupDB, a container runtime that keeps each container's spec label and -// the arguments it was run with, and a service manager that notes, at each restart, the groups the account holds -// in the database then — which is what the machine's own manager gives the unit's new process. -type grantMachine struct { - *groupDB - containers map[string]string - runs map[string][]string - restarts []string -} - -func newGrantMachine(account string, in []string, exists ...string) *grantMachine { - g := newGroupDB(in, exists...) - g.account = account - return &grantMachine{groupDB: g, containers: map[string]string{}, runs: map[string][]string{}} -} - -func (m *grantMachine) run(ctx context.Context, name string, args ...string) (string, error) { - a := oldShape(name, args) - switch name { - case "docker": - m.asked = append(m.asked, name+" "+strings.Join(a, " ")) - switch a[0] { - case "info": - return "27.0\n", nil - case "container": - spec, there := m.containers[a[len(a)-1]] - if !there { - return "", errors.New("no such container") - } - return "true\t" + spec, nil - case "rm": - delete(m.containers, a[len(a)-1]) - case "run": - var named, spec string - for i := 0; i+1 < len(a); i++ { - switch { - case a[i] == "--name": - named = a[i+1] - case a[i] == "--label" && strings.HasPrefix(a[i+1], specLabel+"="): - spec = strings.TrimPrefix(a[i+1], specLabel+"=") - } - } - m.containers[named] = spec - m.runs[named] = a - return "deadbeef\n", nil - } - return "", nil - case "systemctl": - m.asked = append(m.asked, name+" "+strings.Join(a, " ")) - if a[0] == "restart" { - m.restarts = append(m.restarts, a[1]+": "+m.groups()) - } - return "", nil - } - return m.groupDB.run(ctx, name, args...) -} - -// groupOfPaths stands in for the machine's group database and the paths' groups for the length of a test. -func groupOfPaths(t *testing.T, groups map[string]PathGroup) { - t.Helper() - was := GroupOfPath - GroupOfPath = func(path string) (PathGroup, error) { - if _, err := os.Stat(path); err != nil { - return PathGroup{}, err - } - g, ok := groups[path] - if !ok { - return PathGroup{}, errors.New("no group in this test for " + path) - } - return g, nil - } - t.Cleanup(func() { GroupOfPath = was }) - rootGroupsAre(t, map[int]string{0: "root", 10: "wheel", 968: "docker"}) -} - -// rootGroupsAre stands in for the numbers this machine gives the groups whose members act as root. -func rootGroupsAre(t *testing.T, ids map[int]string) { - t.Helper() - was := RootGroupIDs - RootGroupIDs = func() map[int]string { return ids } - t.Cleanup(func() { RootGroupIDs = was }) -} - -func accessGranting(id, path, account string, containers ...string) string { - grant := `"grant":{"account":"` + account + `"` - if len(containers) > 0 { - grant += `,"containers":["` + strings.Join(containers, `","`) + `"]` - } - return `{"id":"` + id + `","type":"access","path":"` + path + `","mode":"read-write",` + grant + `}}` -} - -func mounting(id, path string) string { - return `{"id":"` + id + `","type":"container","name":"` + strings.ReplaceAll(id, ".", "-") + `","image":"` + pinned + - `","volumes":["` + path + `:/library"]}` -} - -func applyGrants(t *testing.T, m *grantMachine, known store.State, resources ...string) (Report, store.State, error) { - t.Helper() - return Apply(context.Background(), archHost(t), parse(t, `{"declaration":1,"resources":[`+ - strings.Join(resources, ",")+`]}`), known, store.OriginDeclared, m.run, nil, nil) -} - -func statOf(t *testing.T, path string) string { - t.Helper() - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - uid, gid, _ := OwnerOf(info) - return fmt.Sprintf("%s %d:%d %s", info.Mode(), uid, gid, info.ModTime()) -} - -func TestAReadWriteAccessGrantsItsPathsGroupToTheContainerAndTheAccountAndLeavesThePath(t *testing.T) { - library := t.TempDir() - if err := os.WriteFile(filepath.Join(library, "album"), []byte("x"), 0o640); err != nil { - t.Fatal(err) - } - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - before, beforeInside := statOf(t, library), statOf(t, filepath.Join(library, "album")) - m := newGrantMachine("operator", []string{"operator"}, "media") - - report, state, err := applyGrants(t, m, store.State{}, - accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(" "+m.groups()+" ", " media ") { - t.Fatalf("the account was not put in the path's group: %q", m.groups()) - } - if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") { - t.Fatalf("the container was not created with the path's group: %s", run) - } - o := outcomeOf(report, "importer.music") - if o.Action != "updated" || !strings.Contains(o.Detail, "grants media (gid 1001)") || - !strings.Contains(o.Detail, "put operator in media") { - t.Errorf("the access did not say what it granted: %+v", o) - } - if rec, _ := state.Find("importer.music"); rec.User != "operator" || strings.Join(rec.Groups, " ") != "media" { - t.Errorf("the access did not record the group it put the account in: %+v", rec) - } - // The path, and what is under it, exactly as they were (ADR 0312 §2). - if after := statOf(t, library); after != before { - t.Errorf("the accessed path changed: %s, was %s", after, before) - } - if after := statOf(t, filepath.Join(library, "album")); after != beforeInside { - t.Errorf("a file under the accessed path changed: %s, was %s", after, beforeInside) - } - for _, asked := range m.asked { - for _, verb := range []string{"chown", "chmod", "chgrp", "setfacl"} { - if strings.HasPrefix(asked, verb) || strings.Contains(asked, " "+verb+" ") { - t.Errorf("the engine asked the machine to change something: %q", asked) - } - } - } - - // Applied again: nothing moves, and nothing is recreated. - m.runs = map[string][]string{} - report, _, err = applyGrants(t, m, state, - accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if o := outcomeOf(report, "importer.music"); o.Action != "unchanged" { - t.Errorf("a second apply moved the access: %+v", o) - } - if len(m.runs) != 0 { - t.Errorf("a second apply recreated the container: %v", m.runs) - } -} - -func TestAReadAccessGrantsNothing(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - - // A grant on a read access is refused whole: the controller asked for something the decision forbids. - _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"player.music","type":"access","path":"` + - library + `","mode":"read","grant":{"account":"operator"}}]}`)) - if err == nil || !strings.Contains(err.Error(), "a read access grants nothing") { - t.Fatalf("a grant on a read access was not refused by name: %v", err) - } - - m := newGrantMachine("operator", []string{"operator"}, "media") - report, state, err := applyGrants(t, m, store.State{}, - `{"id":"player.music","type":"access","path":"`+library+`","mode":"read"}`, mounting("player.app", library)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("a read access put the account in a group: %q", m.groups()) - } - if run := strings.Join(m.runs["player-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("a read access gave its container a group: %s", run) - } - if o := outcomeOf(report, "player.music"); o.Action != "unchanged" { - t.Errorf("a read access moved something: %+v", o) - } - if rec, _ := state.Find("player.music"); len(rec.Groups) != 0 || rec.User != "" { - t.Errorf("a read access recorded a grant: %+v", rec) - } -} - -func TestAnAccessWhoseGroupCannotWriteGrantsNothingAndSaysTheOperatorDecides(t *testing.T) { - spool := t.TempDir() - groupOfPaths(t, map[string]PathGroup{spool: {GID: 1002, Name: "downloads", Writable: false}}) - before := statOf(t, spool) - m := newGrantMachine("operator", nil, "downloads") - - report, _, err := applyGrants(t, m, store.State{}, - accessGranting("nzbget.downloads", spool, "operator", "nzbget.app"), mounting("nzbget.app", spool)) - if err != nil { - t.Fatal(err) - } - o := outcomeOf(report, "nzbget.downloads") - if !strings.Contains(o.Detail, "cannot write it, so nothing is granted") || !strings.Contains(o.Detail, "the operator decides") || - !strings.Contains(o.Detail, "nothing was changed") { - t.Errorf("the access did not say its group cannot write and that the operator decides: %+v", o) - } - // Nothing is given (ADR 0312 §2): no account group, no container group. - if strings.Contains(m.groups(), "downloads") { - t.Errorf("a group that cannot write was granted to the account: %q", m.groups()) - } - if run := strings.Join(m.runs["nzbget-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("a group that cannot write was granted to the container: %s", run) - } - if after := statOf(t, spool); after != before { - t.Errorf("the engine changed a path whose group cannot write: %s, was %s", after, before) - } - // The words the look states it unhealthy with: the remedy is the operator's, and the path is never in them. - words := CannotWriteWords(PathGroup{GID: 1002, Name: "downloads"}) - if !strings.Contains(words, "the operator's to decide") || strings.Contains(words, spool) { - t.Errorf("the unhealthy reason is %q", words) - } -} - -func TestTheRootGroupIsNeverGranted(t *testing.T) { - for _, g := range []PathGroup{{GID: 0, Name: "root", Writable: true}, {GID: 968, Name: "docker", Writable: true}, - {GID: 10, Name: "wheel", Writable: true}, {GID: 42, Name: "shadow", Writable: true}, - // A number the root groups have here under another name — judged by the number, never the name. - {GID: 968, Name: "containers", Writable: true}, - // A number the machine names nothing for: unjudged, so never granted, to a container either. - {GID: 4242, Name: "", Writable: true}} { - want := g.Name - if want == "" || want == "containers" { - want = fmt.Sprintf("gid %d", g.GID) - } - t.Run(fmt.Sprintf("%d %s", g.GID, g.Name), func(t *testing.T) { - shared := t.TempDir() - groupOfPaths(t, map[string]PathGroup{shared: g}) - m := newGrantMachine("operator", nil, g.Name) - - report, _, err := applyGrants(t, m, store.State{}, - accessGranting("importer.shared", shared, "operator", "importer.app"), mounting("importer.app", shared)) - if err == nil { - t.Fatal("a read-write access to a path of a root-granting group applied") - } - var failed *Error - if !errors.As(err, &failed) || failed.Resource != "importer.shared" || - !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never") { - t.Errorf("the refusal does not name the access and the group: %v", err) - } - if g.Name != "" && strings.Contains(m.groups(), g.Name) { - t.Errorf("the account was put in %s", g.Name) - } - if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") { - t.Errorf("the container was given a root-granting group: %s", run) - } - _ = report - }) - } -} - -func TestAGrantedGroupIsTakenBackUnlessStillAskedForOrHeldBefore(t *testing.T) { - music, movies := t.TempDir(), t.TempDir() - groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}, - movies: {GID: 1001, Name: "media", Writable: true}}) - - t.Run("no longer asked", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - report, state, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())) - if err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("the group the mesh granted was not taken back: %q", m.groups()) - } - if o := outcomeOf(report, "importer.music"); o.Action != "restored" || !strings.Contains(o.Detail, "taken out of media") { - t.Errorf("the removal did not say it took the group back: %+v", o) - } - if _, still := state.Find("importer.music"); still { - t.Error("the access is still recorded") - } - }) - - t.Run("still asked by another access", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), - accessGranting("sonarr.movies", movies, "operator")) - if err != nil { - t.Fatal(err) - } - report, _, err := applyGrants(t, m, state, accessGranting("sonarr.movies", movies, "operator")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group another access still grants was taken back: %q", m.groups()) - } - if o := outcomeOf(report, "importer.music"); !strings.Contains(o.Detail, "still asks for") { - t.Errorf("the removal did not say why the group stayed: %+v", o) - } - }) - - t.Run("still asked by a user resource", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator"}, "media") - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - if _, _, err := applyGrants(t, m, state, - `{"id":"player.account","type":"user","name":"operator","groups":["media"]}`); err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group a user resource asks for was taken back: %q", m.groups()) - } - }) - - t.Run("held before", func(t *testing.T) { - m := newGrantMachine("operator", []string{"operator", "media"}) - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator")) - if err != nil { - t.Fatal(err) - } - if rec, _ := state.Find("importer.music"); len(rec.Groups) != 0 { - t.Fatalf("a group the account held before was recorded as the mesh's: %+v", rec) - } - if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Errorf("a group the account held before the mesh was taken back: %q", m.groups()) - } - }) -} - -func TestAContainerIsRecreatedWithTheGroupItIsGranted(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - m := newGrantMachine("operator", nil, "media") - - // Running before its access granted anything: a controller older than the grant. - _, state, err := applyGrants(t, m, store.State{}, - `{"id":"importer.music","type":"access","path":"`+library+`","mode":"read-write"}`, mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") { - t.Fatalf("a container granted nothing was given a group: %s", run) - } - report, _, err := applyGrants(t, m, state, - accessGranting("importer.music", library, "", "importer.app"), mounting("importer.app", library)) - if err != nil { - t.Fatal(err) - } - o := outcomeOf(report, "importer.app") - if o.Action != "updated" || !strings.Contains(o.Detail, "recreated to hold the groups") { - t.Errorf("the container was not recreated for its group: %+v", o) - } - if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") { - t.Errorf("the recreated container does not hold the group: %s", run) - } -} - -func TestTheToolRunnerIsRestartedOnceItsAccountHoldsTheGroup(t *testing.T) { - me, err := user.Current() - if err != nil { - t.Skip("no current account to run the tool runner as") - } - units, bundles := t.TempDir(), t.TempDir() - wasUnits, wasBundles := unitDir, daemonRoot - unitDir, daemonRoot = units, bundles - t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles }) - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - m := newGrantMachine(me.Username, nil, "media") - - body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"}) - runner := `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) + - `","digest":"` + digest + `","run":["node","main.js"],"user":"` + me.Username + - `","restart-on":["importer.music"]}` - // Running before the grant. - _, state, err := applyGrants(t, m, store.State{}, runner) - if err != nil { - t.Fatal(err) - } - m.restarts = nil - report, state, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner) - if err != nil { - t.Fatal(err) - } - if len(m.restarts) != 1 || !strings.HasPrefix(m.restarts[0], "node-tools.service: ") || - !strings.Contains(m.restarts[0], "media") { - t.Fatalf("the tool runner was not restarted once its account held the group: %v", m.restarts) - } - if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "importer.music") { - t.Errorf("the restart does not say what it was for: %+v", o) - } - // And not again while nothing moves. - m.restarts = nil - if _, _, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner); err != nil { - t.Fatal(err) - } - if len(m.restarts) != 0 { - t.Errorf("the tool runner was restarted with nothing moved: %v", m.restarts) - } -} - -// sessionReader is an account whose own service manager runs, holding the groups it began with. -type sessionReader struct{ database, session []string } - -func (s sessionReader) InDatabase(context.Context, string) ([]string, error) { return s.database, nil } -func (s sessionReader) Session(_ context.Context, _ string, groups []string) (accounts.Session, error) { - has := map[string]bool{} - for _, g := range s.session { - has[g] = true - } - return accounts.Session{Running: true, Has: has}, nil -} -func (s sessionReader) Escalation(context.Context, string, []string) ([]string, error) { - return nil, nil -} - -func TestAnAccountsOwnSessionIsSaidReloginNeededForAGrantedGroup(t *testing.T) { - library := t.TempDir() - groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}}) - d := parse(t, `{"declaration":1,"resources":[`+accessGranting("importer.music", library, "operator")+`]}`) - - as := GrantedAccounts(d, nil) - if len(as) != 1 || as[0].Module != "importer" || as[0].Name != "operator" || strings.Join(as[0].Groups, " ") != "media" { - t.Fatalf("the account the access grants is not judged: %+v", as) - } - j := accounts.New(sessionReader{database: []string{"operator", "media"}, session: []string{"operator"}}) - j.Set(as) - st, _ := j.Look(context.Background()) - if len(st.Accounts) != 1 || !strings.HasPrefix(st.Accounts[0].Reason, accounts.ReasonRelogin) { - t.Errorf("a session of the account's own manager that began before the grant is not said relogin needed: %+v", - st.Accounts) - } - // A read access, or one refused, asks nothing of the judge. - groupOfPaths(t, map[string]PathGroup{library: {GID: 0, Name: "root", Writable: true}}) - if as := GrantedAccounts(d, nil); len(as) != 0 { - t.Errorf("a refused grant is judged: %+v", as) - } -} - -// An access going while a user resource still asks for its group hands the group over: the user resource records it -// as the mesh's, and takes it back when it stops asking too (novox/hq ADR 0312 §5, ADR 0252 §3). -func TestAGroupStillAskedForIsHandedOverAndTakenBackWhenTheLastAskerGoes(t *testing.T) { - music := t.TempDir() - groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}}) - player := `{"id":"player.account","type":"user","name":"operator","groups":["media"]}` - m := newGrantMachine("operator", []string{"operator"}, "media") - - _, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), player) - if err != nil { - t.Fatal(err) - } - if rec, _ := state.Find("player.account"); len(rec.Groups) != 0 { - t.Fatalf("the user resource recorded a group the access put the account in: %+v", rec) - } - _, state, err = applyGrants(t, m, state, player) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(m.groups(), "media") { - t.Fatalf("a group the user resource still asks for was taken back: %q", m.groups()) - } - if rec, _ := state.Find("player.account"); strings.Join(rec.Groups, " ") != "media" { - t.Fatalf("the group was not handed to the user resource that still asks for it: %+v", rec) - } - if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil { - t.Fatal(err) - } - if strings.Contains(m.groups(), "media") { - t.Errorf("the group stayed after the last resource asking for it went: %q", m.groups()) - } -} - -// A grant's account is an argument to id, usermod and gpasswd, and a name that starts with a dash would be read as -// an option by each. -func TestAGrantsAccountMayNotReadAsAnOption(t *testing.T) { - for _, name := range []string{"-ogroups", "--help", "a b", "a:b"} { - _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` + - `"mode":"read-write","grant":{"account":"` + name + `"}}]}`)) - if err == nil || !strings.Contains(err.Error(), "not starting with a dash") { - t.Errorf("the account %q was accepted: %v", name, err) - } - } - if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` + - `"mode":"read-write","grant":{"account":"jochen"}}]}`)); err != nil { - t.Errorf("an ordinary account was refused: %v", err) - } -} - -// The scheduler reads no path issue 339's guard refuses, as the apply reads none. -func TestAScheduledStepIsArmedWithoutReadingARefusedPath(t *testing.T) { - var read []string - was := GroupOfPath - GroupOfPath = func(path string) (PathGroup, error) { - read = append(read, path) - return PathGroup{GID: 1001, Name: "media", Writable: true}, nil - } - t.Cleanup(func() { GroupOfPath = was }) - rootGroupsAre(t, map[int]string{0: "root"}) - d := parse(t, `{"declaration":1,"resources":[`+accessGranting("m.etc", "/etc", "", "m.step")+`,`+ - `{"id":"m.step","type":"container","name":"step","image":"`+pinned+`","schedule":"0 3 * * *","volumes":["/etc:/x"]}]}`) - if got := ContainerGroupsOf(d); len(got["m.step"]) != 0 { - t.Errorf("a step was armed with the group of a refused path: %v", got) - } - if len(read) != 0 { - t.Errorf("the scheduler read a path the guard refuses: %v", read) - } -} - -// The group database is read as name to number, so a number carrying a root group's name under any other name -// is still that number. -func TestGroupEntriesAreReadByNumber(t *testing.T) { - got := groupEntries("root:x:0:\n# a comment\ndocker:x:968:operator\ncontainers:x:968:\nbroken\n") - if got["root"] != 0 || got["docker"] != 968 || got["containers"] != 968 || len(got) != 3 { - t.Errorf("read %v", got) - } -} diff --git a/internal/apply/identities.go b/internal/apply/identities.go new file mode 100644 index 0000000..5fb229f --- /dev/null +++ b/internal/apply/identities.go @@ -0,0 +1,253 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Every module is given an identity of its own (novox/hq ADR 0325 §3, §11, issue 567). +// +// The controller allocates each module a user and a group number once, from a range of its own, the same on every +// machine, and declares them as `mesh-`. The engine makes the group and the account with exactly those +// numbers — a system account: no login shell, no home, no password — before anything of the declaration is +// applied, so a directory the module owns can be given to it and a container told to run as it. +// +// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is +// refused for the whole declaration before anything is made or started, naming why — for an account and for an +// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is +// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is. +// +// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the +// engine made the account, the group or both, and an account or a group that was there before is never removed. + +// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the +// runner, which is what a test stands in for. +type accountsTool interface { + CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int) error + CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error + RemoveAccount(ctx context.Context, run system.Runner, name string) error + RemoveGroup(ctx context.Context, run system.Runner, name string) error +} + +// entry is one line the user or group database gives for a key: its name and its numbers. +type entry struct { + name string + id, prim int +} + +// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a +// database that does not answer is an error, never read as absent (system.LookUpUser's rule). +func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) { + out, err := run(ctx, "getent", database, key) + if err != nil { + if code, ok := system.ExitCode(err); ok && code == 2 { + return entry{}, false, nil + } + return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err) + } + fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":") + if len(fields) < 3 { + return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key) + } + e := entry{name: fields[0], prim: -1} + if e.id, err = strconv.Atoi(fields[2]); err != nil { + return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key) + } + if database == "passwd" && len(fields) >= 4 { + if e.prim, err = strconv.Atoi(fields[3]); err != nil { + e.prim = -1 + } + } + return e, true, nil +} + +// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before +// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started. +func refuseRootIdentities(d *declaration.Declaration) error { + var problems []string + for _, id := range d.Identities { + if err := rootIdentity(id.UID, id.GID); err != nil { + problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err)) + } + } + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && c.RunAs != nil { + if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil { + problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v", + c.ID, c.RunAs.UID, c.RunAs.GID, err)) + } + } + } + if len(problems) == 0 { + return nil + } + sort.Strings(problems) + return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; ")) +} + +// applyIdentity makes one identity as declared, or refuses it naming whose the name or the numbers are. What it +// made is recorded on known; an identity already right is unchanged. +func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, known *store.State, + run Runner) (Outcome, error) { + out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"} + user, hasUser, err := lookUp(ctx, run, "passwd", id.Name) + if err != nil { + return out, err + } + group, hasGroup, err := lookUp(ctx, run, "group", id.Name) + if err != nil { + return out, err + } + byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID)) + if err != nil { + return out, err + } + byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.GID)) + if err != nil { + return out, err + } + const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0325 §3)" + switch { + case hasUser && (user.id != id.UID || user.prim != id.GID): + return out, fmt.Errorf("the account %s is here with uid %d and group %d, and the mesh allocated uid %d, "+ + "gid %d%s", id.Name, user.id, user.prim, id.UID, id.GID, never) + case hasGroup && group.id != id.GID: + return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d%s", id.Name, + group.id, id.GID, never) + case uidUsed && byUID.name != id.Name: + return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID, + byUID.name, id.Name, never) + case gidUsed && byGID.name != id.Name: + return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.GID, + byGID.name, id.Name, never) + } + rec := madeIdentity(known, id.Name) + rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID + var made []string + if !hasGroup { + if err := tool.CreateSystemGroup(ctx, run, id.Name, id.GID); err != nil { + return out, err + } + rec.MadeGroup = true + made = append(made, fmt.Sprintf("the group (gid %d)", id.GID)) + } + if !hasUser { + if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID); err != nil { + recordIdentity(known, rec) + return out, err + } + rec.MadeUser = true + made = append(made, fmt.Sprintf("the account (uid %d), with no login, no home and no password", id.UID)) + } + if rec.At.IsZero() { + rec.At = time.Now().UTC() + } + recordIdentity(known, rec) + if len(made) > 0 { + out.Action, out.Detail = "created", "made "+strings.Join(made, " and ") + } else { + out.Detail = "here with its numbers" + } + return out, nil +} + +// madeIdentity is what the engine recorded of name, or an empty record. +func madeIdentity(known *store.State, name string) store.MadeIdentity { + for _, m := range known.Identities { + if m.Name == name { + return m + } + } + return store.MadeIdentity{} +} + +// recordIdentity records what was made of one identity, in place of any record before. +func recordIdentity(known *store.State, rec store.MadeIdentity) { + for i, m := range known.Identities { + if m.Name == rec.Name { + known.Identities[i] = rec + return + } + } + if rec.MadeUser || rec.MadeGroup { + known.Identities = append(known.Identities, rec) + } +} + +// removeIdentities removes what the engine made of every identity no longer declared — the account, then the +// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed +// stays recorded, said, for the next apply. +func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State, + run Runner) []Outcome { + wanted := map[string]bool{} + for _, id := range declared { + wanted[id.Name] = true + } + var out []Outcome + var keep []store.MadeIdentity + for _, m := range known.Identities { + if wanted[m.Name] { + keep = append(keep, m) + continue + } + o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"} + var said []string + if m.MadeUser { + if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) { + said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+ + "database did not answer") + m.MadeUser = false + } else if has { + if err := tool.RemoveAccount(ctx, run, m.Name); err != nil { + said = append(said, "the account could not be removed: "+err.Error()) + } else { + m.MadeUser = false + said = append(said, "removed the account") + } + } else { + m.MadeUser = false + } + } + if m.MadeGroup && !m.MadeUser { + if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) { + said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+ + "database did not answer") + m.MadeGroup = false + } else if has { + if err := tool.RemoveGroup(ctx, run, m.Name); err != nil { + said = append(said, "the group could not be removed: "+err.Error()) + } else { + m.MadeGroup = false + said = append(said, "removed the group") + } + } else { + m.MadeGroup = false + } + } + if m.MadeUser || m.MadeGroup { + o.Action = "waiting" + keep = append(keep, m) + } + o.Detail = "no longer declared: " + strings.Join(said, "; ") + out = append(out, o) + } + known.Identities = keep + return out +} + +// toolFor is the accounts tool of a system, or why it has none. +func toolFor(sys system.System) (accountsTool, error) { + if t, ok := sys.(accountsTool); ok { + return t, nil + } + return nil, errors.New("this system makes no module identity: it has no way to make a system account") +} diff --git a/internal/apply/identities_test.go b/internal/apply/identities_test.go new file mode 100644 index 0000000..a31dd09 --- /dev/null +++ b/internal/apply/identities_test.go @@ -0,0 +1,251 @@ +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with +// the numbers the controller allocated, a container told to run as it, and never a root number. + +// accountsMachine is a user and group database the runner answers from, and every command that changed it. +type accountsMachine struct { + users, groups map[string][2]int // name → id, primary gid (a group's second is unused) + ran []string +} + +func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + switch name { + case "getent": + db, key := args[0], args[1] + table := m.users + if db == "group" { + table = m.groups + } + for n, ids := range table { + if n == key || strconv.Itoa(ids[0]) == key { + if db == "group" { + return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil + } + return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::/:/usr/bin/nologin\n", nil + } + } + return "", errors.New("getent exited 2") + case "groupadd": + m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0} + case "useradd": + m.users[args[len(args)-1]] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])} + case "userdel": + delete(m.users, args[0]) + case "groupdel": + delete(m.groups, args[0]) + default: + return "", nil + } + m.ran = append(m.ran, line) + return "", nil +} + +func mustAtoi(s string) int { + n, _ := strconv.Atoi(s) + return n +} + +func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` + + resources + `]}`)) + if err != nil { + t.Fatal(err) + } + return d +} + +// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was +// made — when no longer declared. +func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) { + m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}} + dir := t.TempDir() + d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`) + report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err != nil { + t.Fatal(err) + } + want := []string{ + "groupadd --system --gid 61001 mesh-radarr", + "useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir / --shell /usr/bin/nologin mesh-radarr", + } + if strings.Join(m.ran, "\n") != strings.Join(want, "\n") { + t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n")) + } + if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" { + t.Errorf("the identity's outcome: %+v", o) + } + if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup { + t.Fatalf("what was made is not recorded: %+v", known.Identities) + } + + // Applied again: nothing is made twice. + m.ran = nil + if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil { + t.Fatal(err) + } + if len(m.ran) != 0 { + t.Fatalf("an identity already here was made again: %v", m.ran) + } + + // Undeclared, with a group the machine had before for another identity: only what was made goes. + m.groups["mesh-sonarr"] = [2]int{61002, 0} + other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`, + `{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`) + if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil { + t.Fatal(err) + } + if _, has := m.users["mesh-radarr"]; has { + t.Error("an identity no longer declared kept its account") + } + if _, has := m.groups["mesh-radarr"]; has { + t.Error("an identity no longer declared kept its group") + } + if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup { + t.Error("a group that was there before is recorded as the mesh's") + } +} + +// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the +// module is not attempted; the rest of the machine is. +func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) { + dir := t.TempDir() + for _, tc := range []struct { + users, groups map[string][2]int + says string + }{ + {users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"}, + {users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"}, + {users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"}, + } { + m := &accountsMachine{users: tc.users, groups: tc.groups} + d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+ + `{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err == nil || !strings.Contains(err.Error(), tc.says) { + t.Errorf("not refused naming %s: %v", tc.says, err) + } + if len(m.ran) != 0 { + t.Errorf("something was made for an identity that is another's: %v", m.ran) + } + if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" { + t.Errorf("the module of a refused identity was attempted: %+v", o) + } + if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil { + t.Errorf("the rest of the machine was not applied: %v", err) + } + os.RemoveAll(filepath.Join(dir, "plex")) + } +} + +// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is +// made or started. A number 0 is refused on arrival, by the parser. +func TestNoModuleIdentityIsRoot(t *testing.T) { + was := RootGroupIDs + RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} } + defer func() { RootGroupIDs = was }() + dir := t.TempDir() + for _, d := range []*declaration.Declaration{ + withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`, + `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`), + withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+ + `{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+ + `"identity":{"mode":"run","uid":61001,"gid":10}}`), + } { + m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}} + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") { + t.Errorf("a root group was not refused naming why: %v", err) + } + if len(m.ran) != 0 { + t.Errorf("something was made or started: %v", m.ran) + } + if _, err := os.Stat(filepath.Join(dir, "config")); err == nil { + t.Error("a resource of a declaration naming a root group was applied") + } + } + for _, raw := range []string{ + `{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`, + `{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` + + `"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`, + } { + if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") { + t.Errorf("a number 0 was not refused on arrival: %v", err) + } + } +} + +// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user +// option — and a changed identity is a container recreated. +func TestAContainerIsToldItsModulesIdentity(t *testing.T) { + env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{ + Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}} + run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{ + Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}} + if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") { + t.Errorf("an image told by environment was not: %s", got) + } + if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") { + t.Errorf("an image told by the runtime's user option was not: %s", got) + } + plain := *env + plain.RunAs = nil + moved := *env + moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", + UID: 61009, GID: 61009} + if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) || + containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) { + t.Error("an identity told, or renumbered, does not recreate the container") + } +} + +// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of +// the access, and nothing is granted again. +func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) { + dir := t.TempDir() + groups := "nox media" + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + switch name { + case "id": + return groups + "\n", nil + case "gpasswd": + groups = "nox" + } + return "", nil + } + known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir, + Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}} + d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`) + report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") { + t.Fatalf("the grant's group was not taken back: %v", ran) + } + if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") { + t.Errorf("the take-back is not said: %+v", o) + } + if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 { + t.Errorf("the record still holds the grant's groups: %+v", rec) + } +} diff --git a/internal/apply/keep.go b/internal/apply/keep.go new file mode 100644 index 0000000..000dc1e --- /dev/null +++ b/internal/apply/keep.go @@ -0,0 +1,579 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// A shared folder is kept usable through access entries (novox/hq ADR 0325, issue 567). +// +// Several modules use one folder of the operator's — a library one imports into, one renames in, one plays from — +// each as its own identity. The engine keeps, for every placed folder, one ACL entry per identity that uses it, +// on the folder and as its default for everything made under it, and changes nothing else of the path (§5). +// +// **When it may keep a folder.** Only one that is the mesh's to keep: taken on the operator's warrant (§8), or +// holding no data yet — empty when first seen, or made by the engine (§9). A folder that holds data and was never +// taken is stated waiting, with a wait of kind access, until the operator lets the mesh keep it (§7); one whose disk +// is not there waits for the disk (§9); root's data on its placed filesystem is never kept, and is unhealthy (§12). +// +// **What every apply does.** It judges the folder (assessFolder), makes a missing one where its data lives, and +// sets the users' entries on the folder itself. What lies under a taken folder is the walks' (ReconcileKept): a +// user added has its entries added to everything under it, a user dropped has exactly its own removed, outside +// the apply and bounded. The watch and the sweep keep what is made later usable (§6, §10). + +// keptRecords is where the engine keeps what it knows of each kept or seen folder: the file beside the state in +// the engine, memory in a test. +type keptRecords interface { + All() (map[string]store.KeptFolder, error) + Update(path string, change func(f *store.KeptFolder, had bool) bool) error +} + +// KeptFolders is the engine's record of the folders it keeps. The engine sets the file beside its state. +var KeptFolders keptRecords = NewKeptMemory() + +// keptMemory is a keptRecords held in memory. +type keptMemory struct { + mu sync.Mutex + m map[string]store.KeptFolder +} + +// NewKeptMemory is an empty record held in memory: a test's, or a one-shot apply's. +func NewKeptMemory() keptRecords { return &keptMemory{m: map[string]store.KeptFolder{}} } + +func (k *keptMemory) All() (map[string]store.KeptFolder, error) { + k.mu.Lock() + defer k.mu.Unlock() + out := map[string]store.KeptFolder{} + for p, f := range k.m { + out[p] = f + } + return out, nil +} + +func (k *keptMemory) Update(path string, change func(f *store.KeptFolder, had bool) bool) error { + k.mu.Lock() + defer k.mu.Unlock() + f, had := k.m[path] + f.Path = path + if change(&f, had) { + k.m[path] = f + } else { + delete(k.m, path) + } + return nil +} + +// keptRecord is the record of path, and whether there is one. A record that cannot be read is none: the folder +// is judged afresh, which waits rather than keeps. +func keptRecord(path string) (store.KeptFolder, bool) { + all, err := KeptFolders.All() + if err != nil { + return store.KeptFolder{}, false + } + f, had := all[path] + return f, had +} + +// folderWant is what every access naming one folder asks of it: the accesses, each uid at the widest use any of +// them gives it (ADR 0325 §4: the tool runner's account stands for every module whose tools use it), and whether +// the controller says it was taken. +type folderWant struct { + path string + ids []string + users map[uint32]store.KeptUser + taken bool +} + +// foldersOf is every declared access's folder, by its clean path. A folder the guard refuses is not read (issue +// 339). +func foldersOf(resources []declaration.Resource, refused map[string]error) map[string]*folderWant { + out := map[string]*folderWant{} + for _, r := range resources { + a, ok := r.(*declaration.Access) + if !ok { + continue + } + path := filepath.Clean(a.Path) + if refused[path] != nil { + continue + } + f := out[path] + if f == nil { + f = &folderWant{path: path, users: map[uint32]store.KeptUser{}} + out[path] = f + } + f.ids = append(f.ids, a.ID) + f.taken = f.taken || a.Taken + for _, u := range a.Users { + had, seen := f.users[uint32(u.UID)] + if seen && (had.Use == declaration.AccessReadWrite || u.Use != declaration.AccessReadWrite) { + continue + } + f.users[uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), Use: u.Use, From: u.From} + } + } + return out +} + +// list is the users, by uid. +func (f *folderWant) list() []store.KeptUser { + if f == nil { + return nil + } + out := make([]store.KeptUser, 0, len(f.users)) + for _, u := range f.users { + out = append(out, u) + } + sort.Slice(out, func(i, j int) bool { return out[i].UID < out[j].UID }) + return out +} + +// usersOf is a list of users as each uid and whether it writes. +func usersOf(us []store.KeptUser) map[uint32]bool { + out := map[uint32]bool{} + for _, u := range us { + out[u.UID] = out[u.UID] || u.Use == declaration.AccessReadWrite + } + return out +} + +// sameUsers is whether two lists give the same uids the same use. +func sameUsers(a, b []store.KeptUser) bool { + x, y := usersOf(a), usersOf(b) + if len(x) != len(y) { + return false + } + for uid, rw := range x { + if w, ok := y[uid]; !ok || w != rw { + return false + } + } + return true +} + +// accessName is the module and its own id for an access, from the resource id `.`. +func accessName(id string) (module, name string) { + if m, ok := moduleOf(id); ok { + return m, id[len(m)+1:] + } + return "", id +} + +// KeepWords and DiskWords are what an access wait says the operator gives (ADR 0325 §7): never a path. +func KeepWords(name string) string { return "let the mesh keep the shared folder " + name } +func DiskWords(name string) string { return "mount the disk that holds " + name } + +// The judgements of a folder. +const ( + // folderKeep is a folder the mesh keeps: taken, or holding no data yet. + folderKeep = "keep" + // folderMake is a folder absent under a parent on its placed filesystem, not root's: made, then kept. + folderMake = "make" + // folderWaitKeep is a folder that holds data and was never taken: it waits for the operator's word. + folderWaitKeep = "wait-keep" + // folderWaitDisk is a folder whose disk is not there. + folderWaitDisk = "wait-disk" + // folderRoot is root's data on its placed filesystem: never kept. + folderRoot = "root" +) + +// folderVerdict is the judgement of one folder, with what it read. +type folderVerdict struct { + kind string + // how is why a folder kept is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptMade. + how string + // mount is the mount point that holds what is there. + mount string + // why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link). + why string + err error +} + +// assessFolder judges a folder as an apply and a look both do (ADR 0325 §7, §9, §12): its disk first, then whose +// it is, then whether the mesh may keep it. +func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict { + others := recordedMounts(path) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + m, err := NearestAncestor(path) + if err != nil { + return folderVerdict{err: err} + } + if !m.Info.IsDir() { + return folderVerdict{err: fmt.Errorf("%s is not there, and its nearest existing parent %s is not a "+ + "folder (a file or a link); nothing is made under it", path, m.Ancestor)} + } + pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others) + if pl.Away { + return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} + } + uid, gid, ok := statOwner(m.Info) + if !ok { + return folderVerdict{err: fmt.Errorf("the owner of %s cannot be read on this machine", m.Ancestor)} + } + if why := rootOwned(uint32(uid), uint32(gid)); why != "" { + return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " + + "existing parent is " + why} + } + return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade} + } + if err != nil { + return folderVerdict{err: err} + } + if !info.IsDir() { + return folderVerdict{err: fmt.Errorf("%s is not a directory, and an access is a shared directory", path)} + } + pl := placedOn(ctx, path, path, rec.Mount, others) + if pl.Away { + return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} + } + facts, err := readFolder(path) + if err != nil { + return folderVerdict{err: err} + } + if why := rootOwned(facts.UID, facts.GID); why != "" { + return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} + } + switch { + case rec.Kept: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} + case taken: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant} + case facts.Empty: + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty} + } + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount} +} + +// recordedMounts is every mount point recorded for another kept folder: where a disk was once seen. +func recordedMounts(except string) []string { + all, err := KeptFolders.All() + if err != nil { + return nil + } + var out []string + for p, f := range all { + if p != except && f.Mount != "" && f.Mount != "/" { + out = append(out, f.Mount) + } + } + return out +} + +// RootWords is the reason root's data is stated unhealthy with (ADR 0325 §12): never a path. +func RootWords(why string) string { + return why + ", and root's data is never kept: the operator places the data elsewhere or gives it an owner " + + "(novox/hq ADR 0325 §12)" +} + +// applyAccess is one access, applied (ADR 0325): its folder judged, made when it is missing where its data lives, +// and its users' entries kept on it when the mesh may keep it; waiting otherwise, and failing for root's data. +// Nothing is changed on the folder but the entries the mesh adds and the mask that shows them. +func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, + in inputs) (Outcome, error) { + // What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5). + gave, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) + out, err := applyAccessFolder(ctx, r, in) + if said != "" { + out.Detail = joinDetail(out.Detail, said) + if gave && out.Action == "unchanged" { + // Updated, so what names the access under restart-on — the tool runner — holds the groups it has now. + out.Action = "updated" + } + } + return out, err +} + +// applyAccessFolder is applyAccess's folder. +func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (Outcome, error) { + out := begin(r) + path := filepath.Clean(r.Path) + want := in.folders[path] + users := want.list() + _, name := accessName(r.ID) + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, want != nil && want.taken) + if v.err != nil { + return out, v.err + } + note := "" + if len(r.Grant) > 0 { + note = "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" + } + switch v.kind { + case folderWaitDisk: + out.Action = "waiting" + out.Detail = "the disk that holds it is not there: nothing is made, and nothing mounts it until it is " + + "(novox/hq ADR 0325 §9); waits for the operator to " + DiskWords(name) + note + return out, nil + case folderRoot: + if len(users) == 0 { + out.Action, out.Detail = "unchanged", "present or not, nothing is kept: no identity uses it"+note + return out, nil + } + return out, errors.New(RootWords(v.why) + "; nothing was made or changed") + case folderMake: + made, err := CreatePath(path) + if err != nil { + return out, err + } + out.Action = "created" + out.Detail = fmt.Sprintf("made %d level(s) as its nearest existing parent is: owner, group and mode "+ + "(novox/hq ADR 0325 §9)", made) + if len(users) == 0 { + out.Detail += note + return out, nil + } + case folderWaitKeep: + if len(users) == 0 { + out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"+note + return out, nil + } + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.Want = v.mount, users + return true + }) + out.Action = "waiting" + out.Detail = "it holds data the mesh has not taken: nothing is changed on it until the operator's warrant " + + "(novox/hq ADR 0325 §7, §8); waits for the operator to " + KeepWords(name) + note + return out, nil + } + if len(users) == 0 { + // Nobody uses it any more: what the mesh added goes on the next walk. + dropping := false + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Want = nil + dropping = len(f.Users) > 0 + return had + }) + if out.Action == "" { + out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it" + } + if dropping { + out.Detail += "; the entries the mesh added are taken out by the next walk" + } + out.Detail += note + return out, nil + } + changed, err := keepTop(path, usersOf(users)) + if errors.Is(err, ErrNoACL) { + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.Want = v.mount, users + return true + }) + out.Action = "waiting" + out.Detail = "its filesystem allows no access entries here: the take act lets it, on the operator's word " + + "(novox/hq ADR 0325 §8); waits for the operator to " + KeepWords(name) + note + return out, nil + } + if err != nil { + return out, err + } + var pending string + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + first := !f.Kept + f.Kept, f.Mount, f.Want = true, v.mount, users + if first { + f.How, f.At = v.how, time.Now().UTC() + // A folder that held nothing has nothing under it to walk; one the controller says was taken + // without this engine's record is walked whole. + if v.how == store.KeptEmpty || v.how == store.KeptMade { + f.Users = users + } + } + if !sameUsers(f.Users, f.Want) { + pending = "; the entries under it follow on the next walk" + } + return true + }) + if out.Action == "" { + out.Action = "unchanged" + if changed { + out.Action = "updated" + } + } + out.Detail = joinDetail(out.Detail, "kept for "+usersWords(users)+": its own entries and default, nothing "+ + "else of it changed (novox/hq ADR 0325 §5)"+pending+note) + return out, nil +} + +// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes +// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group. +// Whether any was taken back, and what was said; nothing for an access that granted nothing. +func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted, + handed Handed) (bool, string) { + if previous.User == "" || len(previous.Groups) == 0 { + return false, "" + } + account := previous + account.Target = previous.User + gave, said := giveGroupsBack(ctx, sys, account, run, wanted, handed) + return gave, "the group grant ADR 0312 made is taken back: " + previous.User + " " + said +} + +// usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)". +func usersWords(users []store.KeptUser) string { + words := make([]string, 0, len(users)) + for _, u := range users { + who := u.Identity + if u.From == declaration.FromToolRunner { + who = "the tool runner's " + who + } + words = append(words, who+" ("+u.Use+")") + } + return strings.Join(words, ", ") +} + +// forgetUndeclared marks every recorded folder no declared access names any more: nobody wants its entries, and +// the next walk takes out exactly what the mesh added, then forgets it. +func forgetUndeclared(folders map[string]*folderWant) { + all, err := KeptFolders.All() + if err != nil { + return + } + for path := range all { + if folders[path] != nil { + continue + } + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Want = nil + // A folder never kept, or kept with nobody's entries left, is forgotten now. + return f.Kept && len(f.Users) > 0 + }) + } +} + +// MissingPath is an accessed folder that is not there, and its nearest existing ancestor. +type MissingPath struct { + Ancestor string + // Levels are the names to make below Ancestor, outermost first. + Levels []string + // Info is the ancestor as lstat reads it: a link is not a directory. + Info os.FileInfo +} + +// NearestAncestor reads, for a path that is not there, its nearest existing ancestor by lstat. An error for a +// path that is there, one not stated exactly, or an ancestor that cannot be read. +func NearestAncestor(path string) (MissingPath, error) { + if err := keptPath(path); err != nil { + return MissingPath{}, err + } + if _, err := os.Lstat(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return MissingPath{}, fmt.Errorf("%s is there, or cannot be read: %v", path, err) + } + var levels []string + for p := path; p != "/"; p = filepath.Dir(p) { + levels = append([]string{filepath.Base(p)}, levels...) + parent := filepath.Dir(p) + info, err := os.Lstat(parent) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return MissingPath{}, err + } + return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil + } + return MissingPath{}, errors.New("no part of the path is there") +} + +// accessPaths is every declared access's folder with its id: what no container mounts while it is not there. +func accessPaths(resources []declaration.Resource, refused map[string]error) map[string]string { + out := map[string]string{} + for _, r := range resources { + if a, ok := r.(*declaration.Access); ok && refused[filepath.Clean(a.Path)] == nil { + out[filepath.Clean(a.Path)] = a.ID + } + } + return out +} + +// mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there +// now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026). +func mountsMissing(c *declaration.Container, accesses map[string]string) string { + for _, v := range c.Volumes { + src := mountSource(v) + if !strings.HasPrefix(src, "/") { + continue + } + for path, id := range accesses { + if !atOrBelow(filepath.Clean(src), path) { + continue + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return id + } + } + } + return "" +} + +// WaitingOnAccess is every container of the declaration that waits for an accessed folder not there, read now: +// not created by the apply, so not judged alive or dead — the access says why. +func WaitingOnAccess(d *declaration.Declaration) map[string]bool { + out := map[string]bool{} + if d == nil { + return out + } + accesses := accessPaths(d.Resources, refusedPlaces(d.Resources)) + if len(accesses) == 0 { + return out + } + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" { + out[c.ID] = true + } + } + return out +} + +// The judgements of an access as a look says them. +const ( + VerdictKeep = "keep" + VerdictMake = "make" + VerdictWaitKeep = "wait-keep" + VerdictWaitDisk = "wait-disk" + VerdictRoot = "root" + VerdictBroken = "broken" + // VerdictNobody is a folder no identity uses: nothing is kept, and only a missing disk is said. + VerdictNobody = "nobody" +) + +// AccessJudgement is a look's judgement of an access's folder, and why, in words. +type AccessJudgement struct { + Kind, Why string +} + +// AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the +// controller says it was taken, used whether any identity uses it. A folder the mesh could keep that no apply has +// kept — its filesystem allowing no ACLs — waits as one not taken does. +func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJudgement { + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, taken) + switch { + case v.err != nil: + return AccessJudgement{Kind: VerdictBroken, Why: v.err.Error()} + case v.kind == folderWaitDisk: + return AccessJudgement{Kind: VerdictWaitDisk} + case !used: + return AccessJudgement{Kind: VerdictNobody} + case v.kind == folderRoot: + return AccessJudgement{Kind: VerdictRoot, Why: v.why} + case v.kind == folderMake: + return AccessJudgement{Kind: VerdictMake} + case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken: + return AccessJudgement{Kind: VerdictWaitKeep} + } + return AccessJudgement{Kind: VerdictKeep} +} diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go new file mode 100644 index 0000000..e87498a --- /dev/null +++ b/internal/apply/keeper_linux.go @@ -0,0 +1,319 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The walks under a kept folder, beside the apply (novox/hq ADR 0325 §6, §8, §10). +// +// The take act adds the users' entries to everything under a folder on the operator's warrant; ReconcileKept adds a +// joining user's and removes exactly a dropped user's under a folder already taken; the sweep looks at a bounded +// part of each kept folder on every look, going on where the last stopped, and puts back what an app narrowed. +// One walk at a time per folder: a sweep finds a folder being walked and waits for the next look. + +// folderLocks is one lock per kept folder, held by every walk of it. +var folderLocks sync.Map + +func folderLock(path string) *sync.Mutex { + l, _ := folderLocks.LoadOrStore(path, &sync.Mutex{}) + return l.(*sync.Mutex) +} + +// actRecord is a walk's counts as the record keeps them. +func actRecord(act, warrant, by string, started time.Time, w *aclWalk, err error) *store.KeptAct { + c := w.counts + rec := &store.KeptAct{Act: act, Warrant: warrant, By: by, State: "done", Started: started, Ended: time.Now().UTC(), + Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other, Mounts: c.Mounts} + switch { + case errors.Is(err, ErrWalkBound): + rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on at the next apply" + case err != nil: + rec.State, rec.Why = "failed", err.Error() + case w.Failed != "": + rec.Why = fmt.Sprintf("%d entries could not be given their entries: %s", c.Unrepaired, w.Failed) + } + return rec +} + +// ReconcileKept walks every kept folder whose entries under it are not what the last apply declared: a user that +// joined has its entries added to everything under it, a dropped user — or one whose use changed — has exactly its +// own taken out (ADR 0325 §8). Bounded by ActLimits; a walk stopped is gone on with at the next call. A folder +// nobody uses any more is forgotten once its entries are out. What it did, in words, for the engine to say. +func ReconcileKept(ctx context.Context) []string { + all, err := KeptFolders.All() + if err != nil { + return []string{"the kept folders could not be read: " + err.Error()} + } + var said []string + for path, f := range all { + if ctx.Err() != nil { + break + } + if !f.Kept || sameUsers(f.Users, f.Want) { + continue + } + said = append(said, reconcileOne(path, f)...) + } + return said +} + +func reconcileOne(path string, f store.KeptFolder) []string { + lock := folderLock(path) + lock.Lock() + defer lock.Unlock() + have, want := usersOf(f.Users), usersOf(f.Want) + drop := map[uint32]bool{} + adds := false + for uid, rw := range have { + if w, ok := want[uid]; !ok || w != rw { + drop[uid] = true + } + } + for uid, rw := range want { + if h, ok := have[uid]; !ok || h != rw { + adds = true + } + } + act, visit := "add", keepVisit(want) + switch { + case len(drop) > 0 && adds: + visit = dropThenKeep(drop, want) + case len(drop) > 0: + act, visit = "drop", dropVisit(drop) + } + started := time.Now().UTC() + w, err := walkKept(path, ActLimits, machineStats, nil, visit) + rec := actRecord(act, "", "", started, w, err) + _ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool { + k.Act = rec + if err == nil { + k.Users = f.Want + } + return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0) + }) + return []string{fmt.Sprintf("the kept folder %s: %s walk %s (%d folders and %d files changed, %d had "+ + "them)%s", path, act, rec.State, rec.Folders, rec.Files, rec.Kept, orWhy(rec.Why))} +} + +func orWhy(why string) string { + if why == "" { + return "" + } + return ": " + why +} + +// TakeFolder is the take act (ADR 0325 §8): on the operator's warrant, the users' entries added to the folder of +// the access id and everything under it. It judges the ask against the declaration last applied and the folder as +// it is, sets the filesystem's ACL type where a ZFS dataset allows none yet, and starts the walk: what it answers +// at once, and a channel closed when the walk is through. Refused, nothing is changed. +func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warrant, by string, + run Runner) (<-chan struct{}, string, error) { + if d == nil { + return nil, "", errors.New("this engine has applied no declaration, and takes no folder it was not told of") + } + var access *declaration.Access + for _, r := range d.Resources { + if a, ok := r.(*declaration.Access); ok && a.ID == id { + access = a + } + } + switch { + case access == nil: + return nil, "", fmt.Errorf("the access %s is not declared on this machine", id) + case filepath.Clean(access.Path) != path: + return nil, "", fmt.Errorf("the access %s is declared at another folder than the one asked", id) + } + folders := foldersOf(d.Resources, refusedPlaces(d.Resources)) + want := folders[path] + users := want.list() + if len(users) == 0 { + return nil, "", errors.New("no identity uses the folder, so there is nothing to keep") + } + rec, _ := keptRecord(path) + v := assessFolder(ctx, path, rec, false) + switch { + case v.err != nil: + return nil, "", v.err + case v.kind == folderWaitDisk: + return nil, "", errors.New("the disk that holds the folder is not there") + case v.kind == folderRoot: + return nil, "", errors.New(RootWords(v.why)) + case v.kind == folderMake: + return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody") + } + lock := folderLock(path) + if !lock.TryLock() { + return nil, "", errors.New("a walk of the folder is running; asked again once it is through, the act goes on") + } + said := "" + _, err := keepTop(path, usersOf(users)) + if errors.Is(err, ErrNoACL) { + allowed, aerr := allowACLs(ctx, v.mount, run) + if aerr != nil { + lock.Unlock() + return nil, "", aerr + } + said = allowed + "; " + _, err = keepTop(path, usersOf(users)) + } + if err != nil { + lock.Unlock() + return nil, "", err + } + started := time.Now().UTC() + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + if !f.Kept { + f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil + } + f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users + f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} + return true + }) + done := make(chan struct{}) + go func() { + defer close(done) + defer lock.Unlock() + w, err := walkKept(path, ActLimits, machineStats, nil, keepVisit(usersOf(users))) + rec := actRecord("take", warrant, by, started, w, err) + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Act = rec + if err == nil { + f.Users = users + } + return true + }) + }() + return done, said + "the take act started under the warrant " + warrant + ": the entries of " + + usersWords(users) + " are added to everything under the folder, following no link and entering no other " + + "filesystem", nil +} + +// allowACLs lets the filesystem at mount carry ACLs, inside the take act only (ADR 0325 §8): a ZFS dataset's +// acltype set to posixacl, the dataset read from the machine's mounts. Any other filesystem is refused, naming why. +func allowACLs(ctx context.Context, mount string, run Runner) (string, error) { + source, fstype := mountSourceOf(mount) + if fstype != "zfs" || source == "" { + return "", fmt.Errorf("the filesystem that holds the folder (%s) allows no access entries, and only a ZFS "+ + "dataset's can be let by the act; the operator mounts it with ACLs", orUnknown(fstype)) + } + if _, err := run(ctx, "zfs", "set", "acltype=posixacl", source); err != nil { + return "", fmt.Errorf("the ZFS dataset that holds the folder could not be let carry access entries "+ + "(zfs set acltype=posixacl): %w", err) + } + return "set acltype=posixacl on the ZFS dataset " + source, nil +} + +func orUnknown(s string) string { + if s == "" { + return "of a type the machine does not say" + } + return s +} + +// KeptLook is what one look found of a kept folder (ADR 0325 §10). +type KeptLook struct { + // Short is every user the folder itself gives less than its right, by entry and mask together, in words. + Short []string + // Busy is a walk of the folder running: the sweep waited. + Busy bool + // Sweep is the sweep's place after this look. + Sweep store.KeptSweep + // Repaired is how many entries this look found narrowed and put back; Unrepaired how many it could not. + Repaired, Unrepaired int + // Failed is why the sweep could not look, in words with no path. + Failed string +} + +// LookKept looks at a kept folder once: the folder itself judged for each user, then the sweep's bounded part of +// what is under it, from where the last look stopped, putting back what an app narrowed. The sweep's place is +// recorded; a look that ran out of its bound says so. +func LookKept(path string, users []store.KeptUser, limits WalkLimits) KeptLook { + var out KeptLook + short, err := judgeTop(path, users) + if err != nil { + out.Failed = err.Error() + return out + } + out.Short = short + lock := folderLock(path) + if !lock.TryLock() { + out.Busy = true + rec, _ := keptRecord(path) + out.Sweep = rec.Sweep + return out + } + defer lock.Unlock() + rec, _ := keptRecord(path) + sw := rec.Sweep + w, err := walkKept(path, limits, machineStats, sw.Cursor, keepVisit(usersOf(users))) + out.Repaired, out.Unrepaired = w.counts.Narrowed-w.counts.Unrepaired, w.counts.Unrepaired + sw.Read += w.counts.Read + sw.Narrowed += w.counts.Narrowed + sw.Unrepaired += w.counts.Unrepaired + switch { + case errors.Is(err, ErrWalkBound): + sw.Cursor, sw.Bound = w.Cursor, true + case err != nil: + out.Failed = err.Error() + default: + // Covered whole: the next look starts a new pass, and what this one could not put back is kept until + // the next pass is through. + sw = store.KeptSweep{Covered: time.Now().UTC(), LastUnrepaired: sw.Unrepaired} + } + out.Sweep = sw + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Sweep = sw + return had + }) + return out +} + +// judgeTop is every user the folder itself gives less than its right, read only. +func judgeTop(path string, users []store.KeptUser) ([]string, error) { + var short []string + _, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return nil, err + } + defer unix.Close(fd) + a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + return nil, err + } + for _, u := range users { + need := needOn(u.Use == declaration.AccessReadWrite, true) + if got := effective(a, u.UID, st.Uid, uint32(st.Mode)&0o7777); got&need != need { + short = append(short, fmt.Sprintf("%s has %s of the %s it needs", u.Identity, permWords(got), + permWords(need))) + } + } + return short, nil +} + +// JudgeKept reads, without changing anything, how many entries under a kept folder give a user less than its +// right, within limits: what the access says while an entry an app narrowed is not put back yet (ADR 0325 §6). +func JudgeKept(path string, users []store.KeptUser, limits WalkLimits) (WalkCounts, error) { + um := usersOf(users) + w, err := walkKept(path, limits, machineStats, nil, func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, + error) { + a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + return false, false, false, err + } + n, u := judgeACL(a, um, st, folder) + return false, n, u, nil + }) + return w.counts, err +} diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go new file mode 100644 index 0000000..10f97dd --- /dev/null +++ b/internal/apply/kept_test.go @@ -0,0 +1,589 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/store" +) + +// The engine's half of ADR 0325's checks (novox/hq ADR 0325, "How it is checked", issue 567): a shared folder kept +// through access entries, on a filesystem of the test's own. Nothing here needs root: a file's owner may set its +// ACL, and the identities are numbers no account here has. + +const ( + radarr = 61001 // a container's identity, read-write + plex = 61002 // a container's identity, read + runner = 61003 // the tool runner's account +) + +// aclFolder is a folder of the test's own on a filesystem that carries ACLs, or the test is skipped saying why. +func aclFolder(t *testing.T) string { + t.Helper() + dir := t.TempDir() + fd, err := unix.Open(dir, unix.O_RDONLY|unix.O_DIRECTORY, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + probe := keptACL(minimalACL(0o755), map[uint32]uint16{radarr: 5}) + if err := setACL(fd, aclAccessXattr, probe); errors.Is(err, ErrNoACL) { + t.Skipf("the test's filesystem (%s) carries no POSIX ACLs (ENOTSUP), and every check here reads them", dir) + } else if err != nil { + t.Fatal(err) + } + if err := unix.Fremovexattr(fd, aclAccessXattr); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + return dir +} + +// keeping has the test keep its own record of kept folders, and judge mounts as the test says. +func keeping(t *testing.T, points []string, targets []string) { + t.Helper() + was, wasPoints, wasTargets := KeptFolders, MountPoints, MountTargets + KeptFolders = NewKeptMemory() + MountPoints = func() []string { return append([]string{"/"}, points...) } + MountTargets = func(context.Context) []string { return targets } + t.Cleanup(func() { KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets }) +} + +// accessJSON is an access of module to path with users, each "identity:uid:use:from". +func accessJSON(id, path, mode string, taken bool, users ...string) string { + var us []string + for _, u := range users { + f := strings.Split(u, ":") + us = append(us, fmt.Sprintf(`{"identity":%q,"uid":%s,"use":%q,"from":%q}`, f[0], f[1], f[2], f[3])) + } + return fmt.Sprintf(`{"id":%q,"type":"access","path":%q,"mode":%q,"taken":%v,"users":[%s]}`, id, path, mode, + taken, strings.Join(us, ",")) +} + +// readACLs is a path's access ACL and default ACL (nil when it has none), and its stat. +func readACLs(t *testing.T, path string) (acl, acl, unix.Stat_t) { + t.Helper() + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NOFOLLOW, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + t.Fatal(err) + } + access, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777) + if err != nil { + t.Fatal(err) + } + def, err := getACL(fd, aclDefaultXattr, uint32(st.Mode)&0o7777) + if err != nil { + t.Fatal(err) + } + return access, def, st +} + +// effectiveOn is what uid may do on path, by entry and mask together. +func effectiveOn(t *testing.T, path string, uid uint32) uint16 { + t.Helper() + a, _, st := readACLs(t, path) + return effective(a, uid, st.Uid, uint32(st.Mode)&0o7777) +} + +func applyAccesses(t *testing.T, known store.State, resources ...string) (Report, store.State, error) { + t.Helper() + d := declare(t, strings.Join(resources, ",")) + return Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, noServices, nil, nil) +} + +// Row 5: entries and default entries per identity and use; owner, owning group, owner's and others' bits unchanged; +// the owning group's entry equals its former group bits; the mask equals what the entries need; an entry the mesh +// did not add is untouched. +func TestAKeptFolderCarriesAnEntryPerUserAndNothingElseOfItChanges(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "movies") + if err := os.Mkdir(lib, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lib, 0o750); err != nil { + t.Fatal(err) + } + // The operator's own entry, there before: never the mesh's to change. + fd, _ := unix.Open(lib, unix.O_RDONLY|unix.O_DIRECTORY, 0) + operator := keptACL(minimalACL(0o750), map[uint32]uint16{60099: permRead}) + operator[len(operator)-1].Perm = permRead | permExec // its mask: r-x, as the operator set it + if err := setACL(fd, aclAccessXattr, operator); err != nil { + t.Fatal(err) + } + unix.Close(fd) + _, _, before := readACLs(t, lib) + + report, _, err := applyAccesses(t, store.State{}, + accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container"), + accessJSON("plex.movies", lib, "read", false, "mesh-plex:61002:read:container")) + if err != nil { + t.Fatalf("an empty folder was not kept: %v", err) + } + access, def, after := readACLs(t, lib) + if bits, _ := access.perm(aclUser, radarr); bits != 7 { + t.Errorf("the read-write identity's entry is %s, and it needs rwx", permWords(bits)) + } + if bits, _ := access.perm(aclUser, plex); bits != 5 { + t.Errorf("the read identity's entry is %s, and it needs r-x", permWords(bits)) + } + for _, uid := range []uint32{radarr, plex} { + if _, ok := def.perm(aclUser, uid); !ok { + t.Errorf("uid %d has no default entry: what is made under the folder would not be its", uid) + } + } + if bits, _ := access.perm(aclUser, 60099); bits != permRead { + t.Errorf("the operator's entry was changed to %s", permWords(bits)) + } + if g, _ := access.perm(aclGroupObj, 0); g != 5 { + t.Errorf("the owning group's entry is %s, and its group bits gave r-x", permWords(g)) + } + if m, _ := access.perm(aclMask, 0); m != 7 { + t.Errorf("the mask is %s, and the entries need rwx", permWords(m)) + } + if after.Uid != before.Uid || after.Gid != before.Gid { + t.Error("the folder's owner or owning group changed") + } + if after.Mode&0o707 != before.Mode&0o707 { + t.Errorf("the owner's or others' bits changed: %o to %o", before.Mode&0o777, after.Mode&0o777) + } + if after.Mode&0o070 != 0o070 { + t.Errorf("the group bits show %o, and they show the mask the entries need (rwx)", (after.Mode>>3)&7) + } + rec, _ := keptRecord(lib) + if !rec.Kept || rec.How != store.KeptEmpty { + t.Errorf("the folder was not recorded kept as one that held nothing: %+v", rec) + } + for _, o := range report.Outcomes { + if o.ID == "radarr.movies" && o.Action == "waiting" { + t.Errorf("an empty folder waited for a warrant: %s", o.Detail) + } + } +} + +// Row 4: the runner's entry is the widest use among the modules whose tools use the folder. +func TestTheToolRunnersEntryIsTheWidestUseAnyModuleGivesIt(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "books") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + _, _, err := applyAccesses(t, store.State{}, + accessJSON("reader.books", lib, "read", false, "nox:61003:read:tool-runner"), + accessJSON("importer.books", lib, "read-write", false, "nox:61003:read-write:tool-runner")) + if err != nil { + t.Fatal(err) + } + if got := effectiveOn(t, lib, runner); got != 7 { + t.Fatalf("the tool runner may %s, and one module whose tools use it declared read-write", permWords(got)) + } +} + +// Row 6: a file made at 0666 by one identity is writable by another read-write identity at once; one made at 0644 +// is writable by it after the watch's repair, and before it the judgement says so; with watches exhausted, the +// sweep repairs it; the judgement reads the effective right (entry and mask). +func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "downloads") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + users := []string{"mesh-radarr:61001:read-write:container", "mesh-sab:61004:read-write:container"} + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.downloads", lib, "read-write", false, + users...)); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + + usual := filepath.Join(lib, "usual") + if err := os.WriteFile(usual, nil, 0o666); err != nil { + t.Fatal(err) + } + if got := effectiveOn(t, usual, radarr); got&6 != 6 { + t.Fatalf("a file made at 0666 gives another read-write identity %s at once", permWords(got)) + } + + w, err := NewWatcher() + if err != nil { + t.Skipf("no inotify here: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go w.Run(ctx) + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + + narrow := filepath.Join(lib, "narrow") + f, err := os.OpenFile(narrow, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.Close() + waitFor(t, "the watch widened the mask of a file made at 0644", func() bool { + return effectiveOn(t, narrow, radarr)&6 == 6 + }) + + // Before a repair the judgement says so: a mask narrowed by hand where nothing watches. + w.Set(nil) + if err := os.Chmod(usual, 0o644); err != nil { + t.Fatal(err) + } + c, err := JudgeKept(lib, rec.Want, ActLimits) + if err != nil { + t.Fatal(err) + } + if c.Narrowed == 0 { + t.Fatal("a file whose mask gives a read-write user only read is not said by the judgement") + } + + // Watches run out: the record says so, and the sweep puts it back. + was := MaxWatchedPerFolder + MaxWatchedPerFolder = 0 + defer func() { MaxWatchedPerFolder = was }() + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + if r, _ := keptRecord(lib); !r.OutOfWatches { + t.Error("watches that ran out are not recorded") + } + look := LookKept(lib, rec.Want, SweepLimits) + if look.Repaired == 0 { + t.Errorf("the sweep put nothing back: %+v", look) + } + if got := effectiveOn(t, usual, radarr); got&6 != 6 { + t.Fatalf("after the sweep a read-write identity may %s", permWords(got)) + } + if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { + t.Fatalf("after the sweep %d entries still give a user less than its right", c.Narrowed) + } +} + +func waitFor(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatalf("not within 3s: %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +// Row 8, the engine's part: a folder that holds data waits, and nothing is changed without a warrant; the act adds +// the entries under the folder, follows no link, enters no other filesystem; a dropped access removes exactly its +// own entries. +func TestAFolderThatHoldsDataIsTakenOnlyOnAWarrantAndLetGoEntryByEntry(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "music") + outside := filepath.Join(dir, "outside") + for _, d := range []string{lib, filepath.Join(lib, "album"), outside} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + song := filepath.Join(lib, "album", "song") + if err := os.WriteFile(song, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(lib, "link")); err != nil { + t.Fatal(err) + } + a := accessJSON("lidarr.music", lib, "read-write", false, "mesh-lidarr:61001:read-write:container", + "mesh-plex:61002:read:container") + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "lidarr.music"); o.Action != "waiting" || !strings.Contains(o.Detail, KeepWords("music")) { + t.Fatalf("a folder that holds data did not wait for the operator: %+v", o) + } + if has := effectiveOn(t, lib, radarr); has != 0 { + t.Fatal("the folder was given an entry without a warrant") + } + if strings.Contains(KeepWords("music"), "/") || strings.Contains(DiskWords("music"), "/") { + t.Fatal("a wait's words name a path") + } + + d := declare(t, a) + done, said, err := TakeFolder(context.Background(), d, "lidarr.music", lib, "warrant-7", "the operator", noServices) + if err != nil { + t.Fatalf("the take act was refused: %v", err) + } + <-done + if !strings.Contains(said, "warrant-7") { + t.Errorf("the act does not say its warrant: %s", said) + } + for _, p := range []string{lib, filepath.Join(lib, "album"), song} { + if got := effectiveOn(t, p, radarr); got&6 != 6 { + t.Errorf("after the act the read-write identity may %s", permWords(got)) + } + } + if has := effectiveOn(t, outside, radarr); has != 0 { + t.Error("the act followed a link out of the folder") + } + rec, _ := keptRecord(lib) + if !rec.Kept || rec.How != store.KeptByWarrant || rec.Warrant != "warrant-7" || rec.Act == nil || + rec.Act.State != "done" || rec.Act.Links != 1 { + t.Fatalf("the act is not recorded as done under its warrant, with the link passed by: %+v %+v", rec, rec.Act) + } + + // Kept now: the next apply keeps it without asking, and plex dropping it has exactly plex's entries removed. + if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false, + "mesh-lidarr:61001:read-write:container")); err != nil { + t.Fatal(err) + } + beforeMask, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }() + ReconcileKept(context.Background()) + for _, p := range []string{lib, song} { + a, def, _ := readACLs(t, p) + if _, still := a.perm(aclUser, plex); still { + t.Errorf("a dropped user's entry stayed") + } + if _, still := def.perm(aclUser, plex); still && def != nil { + t.Errorf("a dropped user's default entry stayed") + } + if _, kept := a.perm(aclUser, radarr); !kept { + t.Errorf("the user that stayed lost its entry") + } + } + if m, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }(); m != beforeMask { + t.Errorf("the mask moved on a drop: %s to %s", permWords(beforeMask), permWords(m)) + } +} + +// Row 8: the act enters no other filesystem — a folder whose device is not the kept folder's is passed by. +func TestAWalkEntersNoOtherFilesystem(t *testing.T) { + dir := aclFolder(t) + mount := filepath.Join(dir, "elsewhere") + if err := os.Mkdir(mount, 0o755); err != nil { + t.Fatal(err) + } + inside := filepath.Join(mount, "file") + if err := os.WriteFile(inside, nil, 0o644); err != nil { + t.Fatal(err) + } + // A stand-in for a device the test cannot mount: the folder named elsewhere reads as another filesystem. + stats := walkStats{ + at: func(dirfd int, name string, st *unix.Stat_t) error { + err := unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW) + if name == "elsewhere" { + st.Dev++ + } + return err + }, + fd: unix.Fstat, + } + w, err := walkKept(dir, ActLimits, stats, nil, keepVisit(map[uint32]bool{radarr: true})) + if err != nil { + t.Fatal(err) + } + if w.counts.Mounts != 1 { + t.Errorf("the walk counted %d folders on another filesystem, and there is one", w.counts.Mounts) + } + if got := effectiveOn(t, inside, radarr); got != 0 { + t.Error("the walk entered another filesystem") + } +} + +// Row 9: a missing folder under a mounted dataset is made as its parent; with the dataset not mounted (its empty +// mount point on the root filesystem) nothing is made and the access waits. +func TestAMissingFolderIsMadeWhereItsDataLivesAndWaitsWhereItsDiskIsNot(t *testing.T) { + dir := aclFolder(t) + dataset := filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dataset, 0o750); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, []string{dataset}) + movies := filepath.Join(dataset, "media", "movies") + report, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", movies, "read-write", false, + "mesh-radarr:61001:read-write:container")) + if err != nil { + t.Fatalf("a missing folder under a mounted dataset was not made: %v", err) + } + info, err := os.Stat(movies) + if err != nil { + t.Fatalf("the folder was not made: %v", err) + } + if info.Mode().Perm()&0o707 != 0o700 { + t.Errorf("the folder was made at %o, and its parent is 0750", info.Mode().Perm()) + } + if got := effectiveOn(t, movies, radarr); got != 7 { + t.Errorf("the folder made is not kept: the identity may %s", permWords(got)) + } + if rec, _ := keptRecord(movies); rec.How != store.KeptMade || rec.Mount != dataset { + t.Errorf("not recorded as made on its dataset: %+v", rec) + } + _ = report + + // The dataset not mounted: its mount point an empty folder on the filesystem below. + MountPoints = func() []string { return []string{"/"} } + books := filepath.Join(dataset, "books") + report, _, err = applyAccesses(t, store.State{}, accessJSON("readarr.books", books, "read-write", false, + "mesh-readarr:61001:read-write:container")) + if err != nil { + t.Fatalf("a missing disk failed the apply: %v", err) + } + if o := outcomeOf(report, "readarr.books"); o.Action != "waiting" || !strings.Contains(o.Detail, DiskWords("books")) { + t.Fatalf("a folder whose disk is not there did not wait for it: %+v", o) + } + if _, err := os.Stat(books); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a folder was made where its disk is not") + } + if v := AccessVerdict(context.Background(), books, false, true); v.Kind != VerdictWaitDisk { + t.Errorf("the look judges it %s, not waiting for its disk", v.Kind) + } + // And the folder made before, its dataset gone: its disk is not there, whatever is at its path now. + if v := AccessVerdict(context.Background(), movies, false, true); v.Kind != VerdictWaitDisk { + t.Errorf("a kept folder whose dataset is not mounted is judged %s", v.Kind) + } +} + +// Row 10: a narrowed entry or mask is reported and put back; successive looks cover a folder whole; a look over +// its bound says how far it got. +func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "shows") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + for i := 0; i < 25; i++ { + p := filepath.Join(lib, fmt.Sprintf("episode-%02d", i)) + if err := os.WriteFile(p, nil, 0o666); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, 0o644); err != nil { // an app's chmod narrows the mask + t.Fatal(err) + } + } + // The folder itself narrowed too: put back on the next apply, said by the look until then. + if err := os.Chmod(lib, 0o755); err != nil { + t.Fatal(err) + } + if short, _ := judgeTop(lib, rec.Want); len(short) == 0 { + t.Fatal("a folder whose mask an app narrowed is not said short") + } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if short, _ := judgeTop(lib, rec.Want); len(short) != 0 { + t.Fatalf("the apply did not put the folder back: %v", short) + } + + limits := WalkLimits{Entries: 10, Time: time.Minute} + looks, repaired := 0, 0 + for { + looks++ + look := LookKept(lib, rec.Want, limits) + repaired += look.Repaired + if look.Failed != "" { + t.Fatal(look.Failed) + } + if !look.Sweep.Bound || look.Sweep.Cursor == nil { + break + } + if len(look.Sweep.Cursor) == 0 || look.Sweep.Read == 0 { + t.Fatalf("a look over its bound does not say how far it got: %+v", look.Sweep) + } + if looks > 10 { + t.Fatal("the sweep never covered the folder") + } + } + if looks < 3 { + t.Errorf("26 entries at 10 a look were covered in %d looks", looks) + } + if repaired != 25 { + t.Errorf("the sweep put back %d of 25 narrowed entries", repaired) + } + if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { + t.Errorf("after the looks %d entries still give the user less than its right", c.Narrowed) + } + if r, _ := keptRecord(lib); r.Sweep.Covered.IsZero() { + t.Error("a sweep that covered the folder is not recorded as having done so") + } +} + +// Row 12: a root-owned folder on its placed filesystem, and an absent one under a root-owned parent on it, are +// unhealthy, nothing is made, and no ask is raised. Read from the machine's own root-owned folders, never changed. +func TestRootsDataOnItsPlacedFilesystemIsNeverKept(t *testing.T) { + keeping(t, nil, nil) + // A root-owned folder the placement guard lets an access name: below a protected root, never one. + root := "" + for _, c := range []string{"/srv/http", "/srv/ftp", "/var/empty", "/var/opt"} { + if info, err := os.Lstat(c); err == nil && info.IsDir() { + if uid, _, _ := statOwner(info); uid == 0 { + root = c + break + } + } + } + if root == "" { + t.Skip("this machine has no root-owned folder below /srv or /var to read") + } + absent := filepath.Join(root, "mesh-test-never-made") + for _, path := range []string{root, absent} { + report, _, err := applyAccesses(t, store.State{}, accessJSON("m.data", path, "read-write", false, + "mesh-m:61001:read-write:container")) + if err == nil || !strings.Contains(err.Error(), "root's data is never kept") { + t.Errorf("root's data at %s was not refused for what it is: %v", path, err) + } + if o := outcomeOf(report, "m.data"); o.Action == "waiting" { + t.Errorf("root's data waits, and is never asked about: %+v", o) + } + if v := AccessVerdict(context.Background(), path, false, true); v.Kind != VerdictRoot { + t.Errorf("the look judges root's data at %s %s", path, v.Kind) + } + } + if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a folder was made under root's data") + } + if _, has := keptRecord(root); has { + t.Error("root's data was recorded as a kept folder") + } +} + +// The codec reads back what it writes, in the kernel's order. +func TestAnACLReadsBackAsWritten(t *testing.T) { + a := keptACL(minimalACL(0o640), map[uint32]uint16{radarr: 6, plex: 4}) + b, err := decodeACL(a.encode()) + if err != nil { + t.Fatal(err) + } + if !sameACL(a, b) || b[0].Tag != aclUserObj || b[len(b)-1].Tag != aclOther { + t.Fatalf("read back %+v", b) + } + if err := b.check(); err != nil { + t.Fatal(err) + } + if _, err := decodeACL([]byte{1, 0, 0, 0}); err == nil { + t.Error("a version 1 attribute was read") + } + if m, _ := b.perm(aclMask, 0); m != 6 { + t.Errorf("mask %s, and the entries need rw- and the group had r--", permWords(m)) + } +} diff --git a/internal/apply/mesh_names_test.go b/internal/apply/mesh_names_test.go index 54ff8cd..794911f 100644 --- a/internal/apply/mesh_names_test.go +++ b/internal/apply/mesh_names_test.go @@ -23,7 +23,7 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) { Name: was.Name, Image: was.Image, Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"}, } - if containerSpecReading(was, nil, nil, nil) == containerSpecReading(moved, nil, nil, nil) { + if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) { t.Fatal("a container whose mesh names moved compares equal, so it is never recreated") } @@ -33,14 +33,14 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) { Name: moved.Name, Image: moved.Image, Hosts: []string{moved.Hosts[1], moved.Hosts[0]}, } - if containerSpecReading(moved, nil, nil, nil) != containerSpecReading(reordered, nil, nil, nil) { + if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) { t.Fatal("the same names in another order read as a different container") } // A container the mesh gives no names is unaffected, so nothing is recreated for a field it // does not set. plain := &declaration.Container{Name: "plex", Image: was.Image} - if containerSpecReading(plain, nil, nil, nil) == containerSpecReading(was, nil, nil, nil) { + if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) { return // different for other reasons, which is fine } } diff --git a/internal/apply/placed.go b/internal/apply/placed.go new file mode 100644 index 0000000..d96163b --- /dev/null +++ b/internal/apply/placed.go @@ -0,0 +1,195 @@ +package apply + +import ( + "bufio" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// Whether a shared folder's disk is there (novox/hq ADR 0325 §9). +// +// **The rule the engine keeps.** A folder is placed on the filesystem it is first seen on: the mount point that +// holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the +// kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged +// against that record: held by another mount point than the one recorded while the recorded one is not mounted is +// the disk not there. A recorded mount point that is mounted is the disk there, whatever now holds the folder (a +// mount made deeper is a placement, not a missing disk), and the record follows it. +// +// **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be: +// every mount target in /etc/fstab, every mount point a ZFS dataset names (`zfs list`, when the machine has zfs), +// and every mount point recorded for another kept folder. A target that is the folder or above it, lies below the +// mount point that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that +// holds what is there is the placed one. A mount nothing names in those places (a hand `mount` of a disk that is not +// in fstab, then unmounted before the engine ever saw the folder) is not known to be missing: said here, because the +// folder would then be made on the filesystem below it. + +// MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test +// can stand in for mounts it cannot make. +var MountPoints = func() []string { + raw, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return []string{"/"} + } + return mountinfoPoints(string(raw)) +} + +// MountTargets is every place a disk is meant to be mounted: /etc/fstab's targets and ZFS datasets' mount points, +// read at most once a minute — every look judges every access by them. A variable so a test can stand in. +var MountTargets = func(ctx context.Context) []string { + targets.Lock() + defer targets.Unlock() + if targets.read.IsZero() || time.Since(targets.read) > time.Minute { + targets.list, targets.read = readMountTargets(ctx), time.Now() + } + return append([]string(nil), targets.list...) +} + +var targets struct { + sync.Mutex + list []string + read time.Time +} + +func readMountTargets(ctx context.Context) []string { + var out []string + if raw, err := os.ReadFile("/etc/fstab"); err == nil { + out = append(out, fstabTargets(string(raw))...) + } + if _, err := exec.LookPath("zfs"); err == nil { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if raw, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "mountpoint").Output(); err == nil { + for _, line := range strings.Split(string(raw), "\n") { + if p := strings.TrimSpace(line); strings.HasPrefix(p, "/") { + out = append(out, filepath.Clean(p)) + } + } + } + } + return out +} + +// mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes +// undone. +func mountinfoPoints(raw string) []string { + var out []string + for _, line := range strings.Split(raw, "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + out = append(out, filepath.Clean(unescapeMount(fields[4]))) + } + return out +} + +// mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its +// name. Empty when the machine says nothing of it. +func mountSourceOf(point string) (source, fstype string) { + raw, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return "", "" + } + for _, line := range strings.Split(string(raw), "\n") { + before, after, ok := strings.Cut(line, " - ") + fields, rest := strings.Fields(before), strings.Fields(after) + if !ok || len(fields) < 5 || len(rest) < 2 { + continue + } + if filepath.Clean(unescapeMount(fields[4])) == point { + source, fstype = unescapeMount(rest[1]), rest[0] + } + } + return source, fstype +} + +// fstabTargets reads the second field of each fstab line that mounts somewhere: not swap, not a comment. +func fstabTargets(raw string) []string { + var out []string + sc := bufio.NewScanner(strings.NewReader(raw)) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || !strings.HasPrefix(fields[1], "/") { + continue + } + out = append(out, filepath.Clean(unescapeMount(fields[1]))) + } + return out +} + +// unescapeMount undoes the kernel's and fstab's octal escapes: \040 is a space. +func unescapeMount(s string) string { + if !strings.Contains(s, `\`) { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+4 <= len(s) { + if v, ok := octal3(s[i+1 : i+4]); ok { + b.WriteByte(v) + i += 3 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +func octal3(s string) (byte, bool) { + if len(s) != 3 { + return 0, false + } + v := 0 + for _, c := range s { + if c < '0' || c > '7' { + return 0, false + } + v = v*8 + int(c-'0') + } + return byte(v), v < 256 +} + +// mountHolding is the longest of points that is path or above it: the mount point that holds it. +func mountHolding(path string, points []string) string { + best := "/" + for _, p := range points { + if atOrBelow(path, p) && len(p) > len(best) { + best = p + } + } + return best +} + +// placement is the judgement of where a folder's data lives. +type placement struct { + // Mount is the mount point that holds what is there: the folder, or its nearest existing parent. + Mount string + // Away is that the disk the folder was placed on is not there. + Away bool +} + +// placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent), +// against what was recorded and where disks are meant to be. +func placedOn(ctx context.Context, path, existing, recorded string, others []string) placement { + points := MountPoints() + mounted := map[string]bool{} + for _, p := range points { + mounted[p] = true + } + have := mountHolding(existing, points) + if recorded != "" { + return placement{Mount: have, Away: have != recorded && !mounted[recorded]} + } + for _, t := range append(MountTargets(ctx), others...) { + if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] { + return placement{Mount: have, Away: true} + } + } + return placement{Mount: have} +} diff --git a/internal/apply/plan.go b/internal/apply/plan.go index 9967a2f..6385eb7 100644 --- a/internal/apply/plan.go +++ b/internal/apply/plan.go @@ -112,7 +112,7 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { step.Verb, step.Why = "restore", "no longer declared; the original the mesh wrote over goes back "+ "from "+orphan.Kept+", unless the file was changed since the mesh last wrote it" case orphan.Type == string(declaration.TypeAccess) && orphan.User != "" && len(orphan.Groups) > 0: - // In removeAccess's words (novox/hq ADR 0312 §5). + // In takeBackGranted's words (novox/hq ADR 0325 §5: no account is in a group for data). step.Verb, step.Why = "restore", "no longer declared; the path is left as it is, and "+orphan.User+ " leaves "+strings.Join(orphan.Groups, ", ")+", which the mesh put it in, unless another declared "+ "resource still asks for it" diff --git a/internal/apply/reads_test.go b/internal/apply/reads_test.go index deaf6f3..be2d8fe 100644 --- a/internal/apply/reads_test.go +++ b/internal/apply/reads_test.go @@ -212,7 +212,7 @@ func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *te t.Fatal(err) } d := declare("5432") - legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil, nil) + legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil) known := store.State{} known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: digestOf("DATABASE_PORT=5432\n")}) diff --git a/internal/apply/roots.go b/internal/apply/roots.go new file mode 100644 index 0000000..8dfccb6 --- /dev/null +++ b/internal/apply/roots.go @@ -0,0 +1,147 @@ +package apply + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/user" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/accounts" +) + +// Root's numbers, which no module identity is and whose data is never kept (novox/hq ADR 0325 §11, §12). +// +// A module identity whose uid or gid is 0, or whose gid is a group whose members act as root (accounts.RootGroups, +// the account judge's own closed list), would be a way to root through a container or a shared folder: refused, +// naming why. A shared folder owned by root or such a group is the machine's own data, never kept. + +// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine +// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine +// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number +// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and +// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can +// stand in. +var RootGroupIDs = func() map[int]string { + rootIDs.Lock() + defer rootIDs.Unlock() + if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute { + rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now() + } + return rootIDs.ids +} + +var rootIDs struct { + sync.Mutex + ids map[int]string + at time.Time +} + +func readRootGroupIDs() map[int]string { + out := map[int]string{} + if raw, err := os.ReadFile("/etc/group"); err == nil { + for name, gid := range groupEntries(string(raw)) { + if slices.Contains(accounts.RootGroups, name) { + out[gid] = name + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for _, name := range accounts.RootGroups { + raw, err := exec.CommandContext(ctx, "getent", "group", name).Output() + if err != nil { + continue + } + for found, gid := range groupEntries(string(raw)) { + if found == name { + out[gid] = name + } + } + } + return out +} + +// groupEntries reads group database lines, name:password:gid:members, as name to number. +func groupEntries(raw string) map[string]int { + out := map[string]int{} + for _, line := range strings.Split(raw, "\n") { + fields := strings.Split(strings.TrimSpace(line), ":") + if len(fields) < 3 || strings.HasPrefix(fields[0], "#") { + continue + } + if gid, err := strconv.Atoi(fields[2]); err == nil { + out[fields[0]] = gid + } + } + return out +} + +// GroupName is the name this machine's group database gives a number, empty when none. A variable so a test can +// stand in for groups it cannot make. +var GroupName = func(gid int) string { + if g, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil { + return g.Name + } + return "" +} + +// rootGroup is why gid is a root group here — 0, a number a root group has, or a number named as one — or "". +func rootGroup(gid int) string { + if gid == 0 { + return "group 0" + } + if name, is := RootGroupIDs()[gid]; is { + return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name) + } + if name := GroupName(gid); slices.Contains(accounts.RootGroups, name) { + return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name) + } + return "" +} + +// rootOwned is why a folder owned by uid and gid is root's data (ADR 0325 §12), or "": owned by root, or by a +// root group. +func rootOwned(uid, gid uint32) string { + if uid == 0 { + return "owned by root" + } + if why := rootGroup(int(gid)); why != "" { + return "owned by " + why + } + return "" +} + +// rootIdentity is why a module identity's numbers are refused (ADR 0325 §11), or nil. +func rootIdentity(uid, gid int) error { + if uid <= 0 { + return fmt.Errorf("its uid is %d, and a module's identity is never root (novox/hq ADR 0325 §11)", uid) + } + if why := rootGroup(gid); why != "" { + return fmt.Errorf("its gid is %s, and a module's identity is never in a root group (novox/hq ADR 0325 §11)", why) + } + return nil +} + +// Handed is every group a record that went in this apply put an account in, and another declared resource still +// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group +// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it +// there went first (novox/hq ADR 0252 §3). +type Handed map[string]map[string]bool + +func (h Handed) hand(account, group string) { + if h == nil { + return + } + if h[account] == nil { + h[account] = map[string]bool{} + } + h[account][group] = true +} + +func (h Handed) has(account, group string) bool { return h[account][group] } diff --git a/internal/apply/schedule.go b/internal/apply/schedule.go index 690e032..2d8e969 100644 --- a/internal/apply/schedule.go +++ b/internal/apply/schedule.go @@ -73,9 +73,6 @@ type scheduledJob struct { // hold is the runtime names of the containers held still for the duration of a run, in the // order the step named them (novox/hq ADR 0189). hold []string - // groups is the number of every group its read-write accesses grant it, read when it was armed (novox/hq - // ADR 0312): a fire runs with them as an apply's container would. - groups []int } // heldStillFor is the runtime names of the containers a step holds still, resolved from ids. @@ -134,14 +131,6 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { defer s.mu.Unlock() seen := map[string]bool{} - // The groups each step's read-write accesses grant (novox/hq ADR 0312), read once, only when a step is here. - var granted map[string][]int - for _, r := range d.Resources { - if c, ok := r.(*declaration.Container); ok && c.Schedule != "" { - granted = ContainerGroupsOf(d) - break - } - } for _, r := range d.Resources { c, ok := r.(*declaration.Container) if !ok || c.Schedule == "" { @@ -169,7 +158,7 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { // Nothing to read: a scheduled container may not declare restart-on — it runs to completion // on its cadence rather than staying running to be restarted — so its identity cannot // depend on another resource's content and there is nothing to pass. - spec := containerSpec(c, inputs{containerGroups: granted}) + spec := containerSpec(c, inputs{}) // The runtime stops containers by name; the declaration names them by id. Resolved here, // against the declaration this job was armed from, so a fire never has to look anything up // (novox/hq ADR 0189). The parser has already refused an id that is not a container here. @@ -178,14 +167,12 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { // Unchanged: keep where it is in its cadence, refresh the declaration pointer only. existing.container = c existing.hold = hold - existing.groups = granted[c.Identity()] continue } // New or changed: arm it for the next due minute after now. next, _ := cron.Next(s.clock.Now()) s.jobs[c.Identity()] = &scheduledJob{ id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold, - groups: granted[c.Identity()], } } @@ -295,7 +282,7 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) { // one that is not there is the state we want, so its error is ignored — the same as run-once. _, _ = s.run(ctx, cri, "rm", "-f", j.container.Name) - args := foregroundRunArgs(j.container, j.spec, j.groups) + args := foregroundRunArgs(j.container, j.spec) if _, err := s.run(ctx, cri, args...); err != nil { s.log(fmt.Sprintf( "scheduled step %s: run exited non-zero — recorded, and left for the next run "+ diff --git a/internal/apply/vocabulary_test.go b/internal/apply/vocabulary_test.go index 882d056..5fc8257 100644 --- a/internal/apply/vocabulary_test.go +++ b/internal/apply/vocabulary_test.go @@ -447,11 +447,11 @@ func TestAChangedResolverOrAddressIsAChangedContainer(t *testing.T) { base := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00"} withDns := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", Dns: []string{"192.168.203.254"}} withIP := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", IP: "192.168.203.254"} - plain := containerSpecReading(base, nil, nil, nil) - if containerSpecReading(withDns, nil, nil, nil) == plain { + plain := containerSpecReading(base, nil, nil) + if containerSpecReading(withDns, nil, nil) == plain { t.Error("adding a resolver did not change the spec, so it can never reach a running container") } - if containerSpecReading(withIP, nil, nil, nil) == plain { + if containerSpecReading(withIP, nil, nil) == plain { t.Error("adding an address did not change the spec, so it can never reach a running container") } } diff --git a/internal/apply/walk.go b/internal/apply/walk.go new file mode 100644 index 0000000..a48ece9 --- /dev/null +++ b/internal/apply/walk.go @@ -0,0 +1,67 @@ +package apply + +import ( + "errors" + "fmt" + "path/filepath" + "time" +) + +// A bounded walk under a kept folder (novox/hq ADR 0325 §8, §10, issue 567). +// +// **Walked by descriptor.** The folder is opened as a mesh directory is (meshWalk: no link another account could +// make, the folder itself no link), and everything under it is reached from its parent's descriptor with +// O_NOFOLLOW: a link met is passed by, a folder whose device is not the kept folder's is not entered, and an entry +// opened is checked to be the one that was looked at. An entry's ACL is read and written through its own +// descriptor, so a name swapped for a link between the look and the change changes nothing. +// +// **Bounded, and resumed.** A media library is millions of files, and a walk that never ends is a machine that +// cannot say what it did. Each walk reads at most so many entries for at most so long; one that meets its bound +// says how far it got (a cursor, the names of each level below the folder, kept in the engine's state and never +// said) and the next goes on from there, in the order of the names, so successive looks cover a folder whole. + +// WalkLimits bound one walk under a folder: how many entries it reads, and for how long. +type WalkLimits struct { + Entries int + Time time.Duration +} + +// ActLimits bound the take act and the walk that adds or drops a user's entries. One stopped goes on at the next +// apply, from the start: what already has its entries is passed quickly. +var ActLimits = WalkLimits{Entries: 5_000_000, Time: 30 * time.Minute} + +// SweepLimits bound the sweep of one kept folder on one look (ADR 0325 §10): the look must come back within its +// own bound whatever the folder holds. +var SweepLimits = WalkLimits{Entries: 2000, Time: 2 * time.Second} + +// WalkCounts is what one walk did or found. Counts only: never a path below the folder. +type WalkCounts struct { + // Folders and Files are the entries changed; Kept the ones that already had what they need. + Folders, Files, Kept int + // Links are links passed by, Other entries neither a file nor a folder, Mounts folders on another + // filesystem not entered. + Links, Other, Mounts int + // Read is every entry looked at. + Read int + // Narrowed is the entries that gave a user less than its right (judged by entry and mask together) and + // were put back; Unrepaired the ones that could not be — the owner's bits, which the mesh never changes. + Narrowed, Unrepaired int +} + +// ErrWalkBound is a walk that met its bound before it was through: what it did stays, and the cursor says where +// the next one goes on. +var ErrWalkBound = errors.New("the walk met its bound") + +// ErrNoACL is a filesystem that allows no ACLs here (ENOTSUP): a ZFS dataset whose acltype is off, say. The access +// waits for the take act, which sets it where it can (ADR 0325 §8). +var ErrNoACL = errors.New("the filesystem allows no access entries") + +// keptPath refuses a folder that is not stated exactly: absolute and clean, so `..` and a doubled separator +// never name it. +func keptPath(path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return fmt.Errorf("%q is not an access's folder as the engine states it (absolute, no `..`, no doubled or "+ + "trailing separator); nothing was changed", path) + } + return nil +} diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go new file mode 100644 index 0000000..d307ff3 --- /dev/null +++ b/internal/apply/walk_linux.go @@ -0,0 +1,536 @@ +//go:build linux + +package apply + +// The descriptor half of walk.go and acl.go (novox/hq ADR 0325 §5, §8, §9, §10). + +import ( + "errors" + "fmt" + "io" + "os" + "sort" + "time" + + "golang.org/x/sys/unix" +) + +// walkStats is how a walk reads an entry and an open descriptor: the machine's own calls, or a test's stand-in +// for a device it cannot mount. +type walkStats struct { + at func(dirfd int, name string, st *unix.Stat_t) error + fd func(fd int, st *unix.Stat_t) error +} + +var machineStats = walkStats{ + at: func(dirfd int, name string, st *unix.Stat_t) error { + return unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW) + }, + fd: unix.Fstat, +} + +// visitFunc is what a walk does to one folder or file, open on fd, whose stat is st: whether it changed it, and +// whether it found the entry narrowed and could not put it back. An error stops the walk. +type visitFunc func(fd int, st unix.Stat_t, folder bool) (changed, narrowed, unrepaired bool, err error) + +// aclWalk is one walk under a kept folder, from the descriptor of the folder itself. +type aclWalk struct { + dev uint64 + deadline time.Time + limit int + stats walkStats + visit visitFunc + counts WalkCounts + // at is the names of the levels below the folder down to the entry being walked; Cursor, once the walk met + // its bound, where the next goes on. + at []string + Cursor []string + // Failed is the first failure on one entry, in words with no path. + Failed string +} + +// openKept opens a kept folder as a mesh directory is opened (no link another account could make, the folder +// itself no link) and starts a walk from it on the folder's own device. +func openKept(path string, limits WalkLimits, stats walkStats, visit visitFunc) (*aclWalk, int, unix.Stat_t, error) { + var st unix.Stat_t + if err := keptPath(path); err != nil { + return nil, -1, st, err + } + mw := &meshWalk{want: path} + fd, err := mw.open() + if errors.Is(err, errNotThereYet) { + return nil, -1, st, errors.New("the folder is not there; nothing was changed") + } + if err != nil { + return nil, -1, st, fmt.Errorf("the folder is not walked: %v", err) + } + if mw.At != path { + unix.Close(fd) + return nil, -1, st, errors.New("the folder is reached through a link, and the engine walks no folder " + + "through one; nothing was changed") + } + if err := stats.fd(fd, &st); err != nil { + unix.Close(fd) + return nil, -1, st, fmt.Errorf("the folder cannot be read: %v; nothing was changed", err) + } + return &aclWalk{dev: uint64(st.Dev), deadline: time.Now().Add(limits.Time), limit: limits.Entries, + stats: stats, visit: visit}, fd, st, nil +} + +// walkKept walks path whole, or from resume, with visit: the walk, with what it did and the cursor where it +// stopped when it met its bound (with ErrWalkBound), and the walk's own error otherwise. +func walkKept(path string, limits WalkLimits, stats walkStats, resume []string, visit visitFunc) (*aclWalk, error) { + w, fd, st, err := openKept(path, limits, stats, visit) + if err != nil { + return &aclWalk{}, err + } + return w, w.dir(fd, st, resume) +} + +// read counts one entry, and says the bound when it is met, keeping where the walk stood as the cursor. +func (w *aclWalk) read() error { + if w.counts.Read >= w.limit || time.Now().After(w.deadline) { + w.Cursor = append([]string(nil), w.at...) + return ErrWalkBound + } + w.counts.Read++ + return nil +} + +// count adds one visit's result. +func (w *aclWalk) count(folder, changed, narrowed, unrepaired bool) { + switch { + case changed && folder: + w.counts.Folders++ + case changed: + w.counts.Files++ + default: + w.counts.Kept++ + } + if narrowed { + w.counts.Narrowed++ + } + if unrepaired { + w.counts.Unrepaired++ + } +} + +// dir is one folder, open on fd, whose stat is st: the folder itself (unless the walk resumes below it, which an +// earlier walk visited), then every entry in it in the order of their names. It closes fd. +func (w *aclWalk) dir(fd int, st unix.Stat_t, resume []string) error { + f := os.NewFile(uintptr(fd), "") + defer f.Close() + if len(resume) == 0 { + if err := w.read(); err != nil { + return err + } + if err := w.visited(fd, st, true); err != nil { + return err + } + } + names, err := f.Readdirnames(-1) + if err != nil { + return fmt.Errorf("a folder could not be read: %v", err) + } + sort.Strings(names) + for _, name := range names { + var below []string + if len(resume) > 0 { + if name < resume[0] { + continue + } + if name == resume[0] { + below = resume[1:] + } + resume = nil + } + w.at = append(w.at, name) + err := w.entry(fd, name, below) + w.at = w.at[:len(w.at)-1] + if err != nil { + return err + } + } + return nil +} + +// entry is one name in the folder open on dirfd: a link passed by, a folder on this device walked, a file +// visited through its own descriptor, anything else left. +func (w *aclWalk) entry(dirfd int, name string, resume []string) error { + var st unix.Stat_t + if err := w.stats.at(dirfd, name, &st); err != nil { + if errors.Is(err, unix.ENOENT) { + return nil + } + return fmt.Errorf("an entry could not be read: %v", err) + } + switch st.Mode & unix.S_IFMT { + case unix.S_IFLNK: + w.counts.Links++ + return nil + case unix.S_IFDIR, unix.S_IFREG: + default: + w.counts.Other++ + return nil + } + if uint64(st.Dev) != w.dev { + w.counts.Mounts++ + return nil + } + isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR + flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK + if isDir { + flags |= unix.O_DIRECTORY + } + fd, err := unix.Openat(dirfd, name, flags, 0) + switch { + case errors.Is(err, unix.ENOENT): + return nil + case errors.Is(err, unix.ELOOP), errors.Is(err, unix.ENOTDIR): + // Swapped for a link, or for something else, after it was looked at: passed by. + w.counts.Links++ + return nil + case err != nil: + return fmt.Errorf("an entry could not be opened: %v", err) + } + var opened unix.Stat_t + if err := w.stats.fd(fd, &opened); err != nil { + unix.Close(fd) + return fmt.Errorf("an entry could not be read: %v", err) + } + if opened.Dev != st.Dev || opened.Ino != st.Ino || opened.Mode&unix.S_IFMT != st.Mode&unix.S_IFMT { + unix.Close(fd) + return errors.New("an entry changed while it was walked, and the walk stopped there") + } + if isDir { + return w.dir(fd, opened, resume) + } + defer unix.Close(fd) + if err := w.read(); err != nil { + return err + } + return w.visited(fd, opened, false) +} + +// visited visits one entry and counts what it did. A filesystem that allows no ACLs stops the walk; any other +// failure on one entry (an immutable file, say) is counted unrepaired, its words kept once, and the walk goes on. +func (w *aclWalk) visited(fd int, st unix.Stat_t, folder bool) error { + changed, narrowed, unrepaired, err := w.visit(fd, st, folder) + if errors.Is(err, ErrNoACL) { + return err + } + if err != nil { + unrepaired = true + if w.Failed == "" { + w.Failed = err.Error() + } + } + w.count(folder, changed, narrowed, unrepaired) + return nil +} + +// getACL reads one of an entry's ACLs through its descriptor: the access ACL, or a folder's default. An access +// ACL that is not there is the one its mode stands for; a default that is not there is nil. +func getACL(fd int, name string, mode uint32) (acl, error) { + buf := make([]byte, 4+8*32) + for { + n, err := unix.Fgetxattr(fd, name, buf) + switch { + case errors.Is(err, unix.ENODATA): + if name == aclAccessXattr { + return minimalACL(mode), nil + } + return nil, nil + case errors.Is(err, unix.ERANGE): + buf = make([]byte, 2*len(buf)) + continue + case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP): + return nil, ErrNoACL + case err != nil: + return nil, fmt.Errorf("an entry's ACL could not be read: %v", err) + } + return decodeACL(buf[:n]) + } +} + +// setACL writes one of an entry's ACLs through its descriptor, refusing one the kernel would. +func setACL(fd int, name string, a acl) error { + if err := a.check(); err != nil { + return err + } + err := unix.Fsetxattr(fd, name, a.encode(), 0) + switch { + case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP): + return ErrNoACL + case err != nil: + return fmt.Errorf("an entry's ACL could not be written: %v", err) + } + return nil +} + +// sameACL is whether a and b hold the same entries, whatever their order. +func sameACL(a, b acl) bool { + return string(a.encode()) == string(b.encode()) +} + +// needs is, for a folder or a file, each kept uid's bits — except its owner's, which its owner's bits give and the +// mesh never changes. +func needs(users map[uint32]bool, owner uint32, folder bool) map[uint32]uint16 { + out := map[uint32]uint16{} + for uid, rw := range users { + if uid != owner { + out[uid] = needOn(rw, folder) + } + } + return out +} + +// keepVisit makes users' entries on each entry (ADR 0325 §5): the access ACL with each uid's entry and a mask +// that covers it, and on a folder the same as its default, so what is made under it inherits them. An entry that +// had them is not written. A user that owns the entry is judged by its owner's bits, never given an entry: an +// owner whose bits fall short is unrepaired, said, and never changed. +func keepVisit(users map[uint32]bool) visitFunc { + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + mode := uint32(st.Mode) & 0o7777 + need := needs(users, st.Uid, folder) + access, err := getACL(fd, aclAccessXattr, mode) + if err != nil { + return false, false, false, err + } + narrowed, unrepaired := judgeACL(access, users, st, folder) + changed := false + if kept := keptACL(access, need); !sameACL(kept, access) { + if err := setACL(fd, aclAccessXattr, kept); err != nil { + return false, narrowed, true, err + } + changed = true + } + if folder { + def, err := getACL(fd, aclDefaultXattr, mode) + if err != nil { + return changed, narrowed, unrepaired, err + } + base := def + if base == nil { + // No default yet: one whose owner's, owning group's and others' entries are the folder's own, as + // setfacl -d derives it. + base = minimalACL(mode) + if g, ok := access.perm(aclGroupObj, 0); ok { + base[1].Perm = g + } + } + // The default is for what anyone makes under the folder, never its owner alone: every user's entry. + all := map[uint32]uint16{} + for uid, rw := range users { + all[uid] = needOn(rw, true) + } + if kept := keptACL(base, all); def == nil || !sameACL(kept, def) { + if err := setACL(fd, aclDefaultXattr, kept); err != nil { + return changed, narrowed, true, err + } + changed = true + } + } + return changed, narrowed, unrepaired, nil + } +} + +// judgeACL is whether an entry with access ACL a gives a user less than its right — by entry and mask together +// (ADR 0325 §6) — and whether that is one the mesh cannot put back: the owner's own bits. +func judgeACL(a acl, users map[uint32]bool, st unix.Stat_t, folder bool) (narrowed, unrepaired bool) { + mode := uint32(st.Mode) & 0o7777 + for uid, rw := range users { + need := needOn(rw, folder) + if effective(a, uid, st.Uid, mode)&need == need { + continue + } + narrowed = true + if uid == st.Uid { + unrepaired = true + } + } + return narrowed, unrepaired +} + +// dropVisit takes exactly the entries of each uid of drop out of each entry's access ACL and a folder's default +// (ADR 0325 §8): no other entry, and not the mask, so nobody else loses a right. +func dropVisit(drop map[uint32]bool) visitFunc { + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + mode := uint32(st.Mode) & 0o7777 + changed := false + for _, name := range []string{aclAccessXattr, aclDefaultXattr} { + if name == aclDefaultXattr && !folder { + continue + } + a, err := getACL(fd, name, mode) + if errors.Is(err, ErrNoACL) || a == nil { + continue + } + if err != nil { + return changed, false, false, err + } + if left := withoutUsers(a, drop); len(left) != len(a) { + if err := setACL(fd, name, left); err != nil { + return changed, false, true, err + } + changed = true + } + } + return changed, false, false, nil + } +} + +// dropThenKeep is drop then keep, on one walk: a user whose use changed, or one gone beside one come. +func dropThenKeep(drop map[uint32]bool, users map[uint32]bool) visitFunc { + d, k := dropVisit(drop), keepVisit(users) + return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) { + c1, _, _, err := d(fd, st, folder) + if err != nil { + return c1, false, false, err + } + c2, n, u, err := k(fd, st, folder) + return c1 || c2, n, u, err + } +} + +// folderEmpty is whether the folder open on fd holds nothing yet: the data the mesh may keep without asking. +func folderEmpty(fd int) (bool, error) { + dup, err := unix.Dup(fd) + if err != nil { + return false, err + } + f := os.NewFile(uintptr(dup), "") + defer f.Close() + names, err := f.Readdirnames(1) + switch { + case len(names) > 0: + return false, nil + case err == nil, errors.Is(err, io.EOF): + return true, nil + } + return false, fmt.Errorf("the folder could not be read: %v", err) +} + +// keepTop keeps users' entries on the kept folder itself (ADR 0325 §5): what every apply does, beside the walks. +func keepTop(path string, users map[uint32]bool) (changed bool, err error) { + w, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, keepVisit(users)) + if err != nil { + return false, err + } + defer unix.Close(fd) + changed, _, _, err = w.visit(fd, st, true) + return changed, err +} + +// FolderFacts is what the engine reads of a folder before keeping it: its owner, group and whether it holds data. +type FolderFacts struct { + UID, GID uint32 + Empty bool +} + +// readFolder reads a folder's facts through the descriptor a mesh directory is opened by. +func readFolder(path string) (FolderFacts, error) { + if err := keptPath(path); err != nil { + return FolderFacts{}, err + } + mw := &meshWalk{want: path} + fd, err := mw.open() + if err != nil { + return FolderFacts{}, err + } + defer unix.Close(fd) + if mw.At != path { + return FolderFacts{}, errors.New("the folder is reached through a link, and the engine keeps no folder " + + "through one") + } + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return FolderFacts{}, err + } + empty, err := folderEmpty(fd) + if err != nil { + return FolderFacts{}, err + } + return FolderFacts{UID: st.Uid, GID: st.Gid, Empty: empty}, nil +} + +// CreatePath makes a missing accessed folder (ADR 0325 §9): each missing level made from the nearest existing +// ancestor down, each with that ancestor's owner, group and mode. No link is followed (the ancestor is reached as +// a mesh directory is, and each level is opened with O_NOFOLLOW), no level is made on another filesystem, no file +// is made, and nothing that was there is changed: a level that appears meanwhile stops it. The caller has judged +// that the ancestor is on the placed filesystem and not root's. How many levels it made. +func CreatePath(path string) (int, error) { + return createPath(path, machineStats) +} + +func createPath(path string, stats walkStats) (int, error) { + m, err := NearestAncestor(path) + if err != nil { + return 0, fmt.Errorf("the folder cannot be made: %v; nothing was changed", err) + } + if !m.Info.IsDir() { + return 0, errors.New("its nearest existing parent is not a folder (a file or a link), and a folder is " + + "made only below a folder; nothing was changed") + } + mw := &meshWalk{want: m.Ancestor} + fd, err := mw.open() + if err != nil { + return 0, fmt.Errorf("the nearest existing parent is not walked: %v; nothing was changed", err) + } + if mw.At != m.Ancestor { + unix.Close(fd) + return 0, errors.New("the nearest existing parent is reached through a link, and nothing is made " + + "through one; nothing was changed") + } + var anc unix.Stat_t + if err := stats.fd(fd, &anc); err != nil { + unix.Close(fd) + return 0, fmt.Errorf("the nearest existing parent cannot be read: %v; nothing was changed", err) + } + if why := rootOwned(anc.Uid, anc.Gid); why != "" { + unix.Close(fd) + return 0, fmt.Errorf("its nearest existing parent is %s, and nothing is made under root's data "+ + "(novox/hq ADR 0325 §12); nothing was changed", why) + } + mode := uint32(anc.Mode) & 0o7777 + made := 0 + for _, name := range m.Levels { + // Made for its maker alone (the engine, as root), so it is nobody else's until it is the ancestor's. + if err := unix.Mkdirat(fd, name, 0o700); err != nil { + unix.Close(fd) + if errors.Is(err, unix.EEXIST) { + return made, errors.New("a level of the folder appeared while it was made, and nothing that is " + + "there is changed; it stopped there") + } + return made, fmt.Errorf("a level could not be made: %v", err) + } + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + unix.Close(fd) + if err != nil { + return made, fmt.Errorf("a level made could not be opened: %v", err) + } + fd = next + made++ + var st unix.Stat_t + if err := stats.fd(fd, &st); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be read: %v", err) + } + if uint64(st.Dev) != uint64(anc.Dev) { + unix.Close(fd) + return made, errors.New("a level made is on another filesystem than its parent, and it stopped there") + } + if st.Uid != anc.Uid || st.Gid != anc.Gid { + if err := unix.Fchown(fd, int(anc.Uid), int(anc.Gid)); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be given its parent's owner: %v", err) + } + } + if err := unix.Fchmod(fd, mode); err != nil { + unix.Close(fd) + return made, fmt.Errorf("a level made could not be given its parent's mode: %v", err) + } + } + unix.Close(fd) + return made, nil +} diff --git a/internal/apply/walk_other.go b/internal/apply/walk_other.go new file mode 100644 index 0000000..c341303 --- /dev/null +++ b/internal/apply/walk_other.go @@ -0,0 +1,55 @@ +//go:build !linux + +package apply + +import ( + "context" + "errors" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The node-engine runs on Linux: a folder is kept only by descriptor and through the kernel's ACL attributes, +// which this system has not. Everything refuses, saying so, and changes nothing. + +var errNoKeeping = errors.New("a shared folder is kept only by descriptor, which this system cannot; nothing was changed") + +// FolderFacts is what the engine reads of a folder before keeping it. +type FolderFacts struct { + UID, GID uint32 + Empty bool +} + +func readFolder(string) (FolderFacts, error) { return FolderFacts{}, errNoKeeping } +func keepTop(string, map[uint32]bool) (bool, error) { return false, errNoKeeping } +func CreatePath(string) (int, error) { return 0, errNoKeeping } +func ReconcileKept(context.Context) []string { return nil } +func JudgeKept(string, []store.KeptUser, WalkLimits) (WalkCounts, error) { + return WalkCounts{}, errNoKeeping +} + +// KeptLook is what one look found of a kept folder. +type KeptLook struct { + Short []string + Busy bool + Sweep store.KeptSweep + Repaired, Unrepaired int + Failed string +} + +func LookKept(string, []store.KeptUser, WalkLimits) KeptLook { + return KeptLook{Failed: errNoKeeping.Error()} +} + +func TakeFolder(context.Context, *declaration.Declaration, string, string, string, string, Runner) (<-chan struct{}, + string, error) { + return nil, "", errNoKeeping +} + +// Watcher watches nothing where a folder is not kept. +type Watcher struct{} + +func NewWatcher() (*Watcher, error) { return nil, errNoKeeping } +func (*Watcher) Set(map[string][]store.KeptUser) {} +func (*Watcher) Run(context.Context) {} diff --git a/internal/apply/watch_linux.go b/internal/apply/watch_linux.go new file mode 100644 index 0000000..e79ce16 --- /dev/null +++ b/internal/apply/watch_linux.go @@ -0,0 +1,292 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "strconv" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/store" +) + +// The watch on each kept folder (novox/hq ADR 0325 §6). +// +// What an identity makes in a kept folder inherits the default entries, and its mask follows the mode its maker +// asked for: a file made at 0644 gives a read-write user only read until its mask is widened. The engine watches +// every folder of each kept tree (inotify: made, moved in, attributes changed) and, on notice, gives the entry its +// users' entries and a mask that covers them — the same keepVisit the walks use, through the entry's own +// descriptor, reached from the kept folder down by O_NOFOLLOW, never through a link and never on another +// filesystem. Notice can be lost (the kernel's queue overflowed) and watches run out (the machine's limit, ENOSPC): +// then the record says so and the sweep finds what was made where no watch is (§10). + +// watchMask is what a watch hears: an entry made or moved in, and an entry's mode or ACL changed. +// It is added through /proc/self/fd, whose link is the folder's own descriptor: following it is what reaches the +// folder that was opened, so the watch takes no IN_DONT_FOLLOW. +const watchMask = unix.IN_CREATE | unix.IN_MOVED_TO | unix.IN_ATTRIB | unix.IN_ONLYDIR + +// MaxWatchedPerFolder bounds the folders watched under one kept folder: beyond it the sweep alone keeps the rest. +var MaxWatchedPerFolder = 100_000 + +// Watcher watches every kept folder's tree. +type Watcher struct { + mu sync.Mutex + fd int + dirs map[int]watchedDir + roots map[string]*watchedRoot +} + +// watchedDir is one watched folder: its kept folder and the names below it. +type watchedDir struct { + root string + rel []string +} + +// watchedRoot is one kept folder watched: its users, its device, and its watches. +type watchedRoot struct { + users map[uint32]bool + dev uint64 + wds []int + out bool +} + +// NewWatcher opens the machine's notice of changes. +func NewWatcher() (*Watcher, error) { + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC) + if err != nil { + return nil, err + } + return &Watcher{fd: fd, dirs: map[int]watchedDir{}, roots: map[string]*watchedRoot{}}, nil +} + +// Set is every kept folder to watch, with its users: one newly kept is watched whole (within +// MaxWatchedPerFolder), one no longer kept is let go, and one whose users changed is repaired with the new ones. +func (w *Watcher) Set(folders map[string][]store.KeptUser) { + w.mu.Lock() + defer w.mu.Unlock() + for path, root := range w.roots { + if _, still := folders[path]; !still { + for _, wd := range root.wds { + _, _ = unix.InotifyRmWatch(w.fd, uint32(wd)) + delete(w.dirs, wd) + } + delete(w.roots, path) + } + } + for path, users := range folders { + if root, had := w.roots[path]; had { + root.users = usersOf(users) + continue + } + root := &watchedRoot{users: usersOf(users)} + w.roots[path] = root + w.watchTree(path, root) + w.say(path, root) + } +} + +// say records how many folders of a kept folder are watched, and whether the watches ran out. +func (w *Watcher) say(path string, root *watchedRoot) { + n, out := len(root.wds), root.out + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + f.Watches, f.OutOfWatches = n, out + return had + }) +} + +// watchTree watches a kept folder and every folder under it on its device, by descriptor: each watch is added +// through /proc/self/fd of the folder's own descriptor, so no name is followed to another place. +func (w *Watcher) watchTree(path string, root *watchedRoot) { + _, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return + } + root.dev = uint64(st.Dev) + w.watchBelow(path, root, fd, nil) +} + +// watchBelow watches the folder open on fd, rel below root, and every folder under it. It closes fd. +func (w *Watcher) watchBelow(path string, root *watchedRoot, fd int, rel []string) { + defer unix.Close(fd) + if root.out || len(root.wds) >= MaxWatchedPerFolder { + root.out = true + return + } + wd, err := unix.InotifyAddWatch(w.fd, "/proc/self/fd/"+strconv.Itoa(fd), watchMask) + if errors.Is(err, unix.ENOSPC) { + root.out = true + return + } + if err != nil { + return + } + root.wds = append(root.wds, wd) + w.dirs[wd] = watchedDir{root: path, rel: append([]string(nil), rel...)} + dup, err := unix.Dup(fd) + if err != nil { + return + } + names, err := readNames(dup) + if err != nil { + return + } + for _, name := range names { + var st unix.Stat_t + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR || + uint64(st.Dev) != root.dev { + continue + } + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + continue + } + w.watchBelow(path, root, next, append(rel, name)) + } +} + +// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder. +func (w *Watcher) Run(ctx context.Context) { + go func() { + <-ctx.Done() + unix.Close(w.fd) + }() + buf := make([]byte, 64*1024) + for { + n, err := unix.Read(w.fd, buf) + if err != nil || n <= 0 { + if ctx.Err() != nil || !errors.Is(err, unix.EINTR) { + return + } + continue + } + for at := 0; at+unix.SizeofInotifyEvent <= n; { + ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[at])) + nameAt := at + unix.SizeofInotifyEvent + name := cString(buf[nameAt : nameAt+int(ev.Len)]) + at = nameAt + int(ev.Len) + if ev.Mask&unix.IN_Q_OVERFLOW != 0 { + w.lost() + continue + } + w.heard(int(ev.Wd), name, ev.Mask) + } + } +} + +// lost is notice lost: every kept folder is said to rely on its sweep until it is watched afresh. +func (w *Watcher) lost() { + w.mu.Lock() + defer w.mu.Unlock() + for path, root := range w.roots { + root.out = true + w.say(path, root) + } +} + +// heard repairs one entry a watch heard of: name in the watched folder wd, or the folder itself. +func (w *Watcher) heard(wd int, name string, mask uint32) { + w.mu.Lock() + dir, ok := w.dirs[wd] + root := w.roots[dir.root] + w.mu.Unlock() + if !ok || root == nil { + return + } + fd, err := openBelow(dir.root, dir.rel, root.dev) + if err != nil { + return + } + defer unix.Close(fd) + visit := keepVisit(root.users) + if name == "" { + var st unix.Stat_t + if unix.Fstat(fd, &st) == nil { + _, _, _, _ = visit(fd, st, true) + } + return + } + var st unix.Stat_t + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != root.dev { + return + } + isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR + if !isDir && st.Mode&unix.S_IFMT != unix.S_IFREG { + return + } + flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK + if isDir { + flags |= unix.O_DIRECTORY + } + efd, err := unix.Openat(fd, name, flags, 0) + if err != nil { + return + } + var opened unix.Stat_t + if unix.Fstat(efd, &opened) != nil || opened.Ino != st.Ino || opened.Dev != st.Dev { + unix.Close(efd) + return + } + _, _, _, _ = visit(efd, opened, isDir) + if isDir && mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 { + // A folder made or moved in is watched, and what was moved in with it is the sweep's. + w.mu.Lock() + w.watchBelow(dir.root, root, efd, append(append([]string(nil), dir.rel...), name)) + w.say(dir.root, root) + w.mu.Unlock() + return + } + unix.Close(efd) +} + +// openBelow opens the folder rel below the kept folder root, each level by O_NOFOLLOW on root's device. +func openBelow(root string, rel []string, dev uint64) (int, error) { + _, fd, _, err := openKept(root, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil) + if err != nil { + return -1, err + } + for _, name := range rel { + next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + unix.Close(fd) + if err != nil { + return -1, err + } + var st unix.Stat_t + if err := unix.Fstat(next, &st); err != nil || uint64(st.Dev) != dev { + unix.Close(next) + return -1, errors.New("a level is on another filesystem") + } + fd = next + } + return fd, nil +} + +// readNames reads every name in the folder open on fd, and closes it. +func readNames(fd int) ([]string, error) { + var names []string + buf := make([]byte, 32*1024) + defer unix.Close(fd) + for { + n, err := unix.ReadDirent(fd, buf) + if err != nil { + return names, err + } + if n <= 0 { + return names, nil + } + _, _, names = unix.ParseDirent(buf[:n], -1, names) + } +} + +func cString(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index e2f6d29..4f4ba92 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -456,8 +456,10 @@ const ( // **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own — // it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A // media library, a download spool is the operator's: it existed before the mesh, several modules -// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The -// host confirms it is there and mounts it; nothing else. +// read and write it at once, and the mesh must not chown, chmod or remove it. The host confirms it +// is there and mounts it, and keeps it usable by the identities that use it through ACL entries it +// adds and nothing else (novox/hq ADR 0325 §5); a missing one is made only below a parent on the +// filesystem the data was placed on (§9). type Access struct { ID string `json:"id"` Type Type `json:"type"` @@ -465,33 +467,44 @@ type Access struct { // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` - // Grant is who the path's own group is granted to, for a read-write access (novox/hq ADR 0312 §1): the - // identities the module's code runs as. The controller says who; the node-engine reads which group owns - // the path when it applies, because only the machine knows, and never changes the path (§2). Absent is a - // controller older than the grant, or an access that grants nothing — and a read access always is one (§3). - Grant *AccessGrant `json:"grant,omitempty"` + // Users are the identities that use the folder, resolved by the controller (novox/hq ADR 0325 §3, §4): each + // module identity whose container mounts it, and the tool runner's account standing for every module whose + // tools use it. The engine keeps one ACL entry per user on the folder and as its default (§5); several + // accesses naming one folder are kept as one, each uid at the widest use any of them gives it. Absent is a + // controller older than ADR 0325, or an access nobody but others' bits reach (§2): nothing is kept. + Users []AccessUser `json:"users,omitempty"` + // Taken is the controller saying the operator's warrant for this folder was carried out (§8): the folder is + // kept although it holds data. The engine also keeps a folder it took itself, on the act, or found empty. + Taken bool `json:"taken,omitempty"` + + // Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted + // so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5). + Grant json.RawMessage `json:"grant,omitempty"` } -// AccessGrant is the identities a read-write access grants its path's group to (novox/hq ADR 0312 §1). -type AccessGrant struct { - // Containers are the module's containers that mount the path, by id: each is created with the group as a - // supplementary group of its process, and recreated when the group moves (§6). - Containers []string `json:"containers,omitempty"` - // Account is the account that runs the module's tools: put in the group as an account group, as ADR 0252 - // §1 puts one, and taken out again as §3 takes back (§5). Empty for a module whose code runs nowhere as - // an account. - Account string `json:"account,omitempty"` +// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4). +const ( + FromContainer = "container" + FromToolRunner = "tool-runner" +) + +// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its +// use, and where it runs. +type AccessUser struct { + Identity string `json:"identity"` + UID int `json:"uid"` + Use string `json:"use"` + From string `json:"from"` } -// UnmarshalJSON reads a grant as strictly as a resource is read: a field the engine does not know is a -// thing the controller believes it asked for, and is refused rather than ignored. -func (g *AccessGrant) UnmarshalJSON(raw []byte) error { - type plain AccessGrant +// UnmarshalJSON reads a user as strictly as a resource is read. +func (u *AccessUser) UnmarshalJSON(raw []byte) error { + type plain AccessUser var p plain if err := strictDecode(raw, &p); err != nil { - return fmt.Errorf("a grant: %w", err) + return fmt.Errorf("an access's user: %w", err) } - *g = AccessGrant(p) + *u = AccessUser(p) return nil } @@ -511,27 +524,41 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } - if a.Grant != nil && a.Mode != AccessReadWrite { - // A read access reads as any account may, and gains no group (novox/hq ADR 0312 §3). - problems = append(problems, where+": a read access grants nothing, and this one names a grant; "+ - "only a read-write access grants its path's group (novox/hq ADR 0312 §3)") - } - if a.Grant != nil { - for _, id := range a.Grant.Containers { - if id == "" { - problems = append(problems, where+": a grant names a container with no id") - } + for _, u := range a.Users { + switch { + case u.Identity == "" || !accountName.MatchString(u.Identity): + problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one", + where, u.Identity)) + case u.UID <= 0: + // Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a + // number below it is no account. + problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+ + "root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID)) } - if a.Grant.Account != "" && !accountName.MatchString(a.Grant.Account) { - // Handed to id, usermod and gpasswd as an argument: a name starting with a dash would be read as an - // option by each of them. - problems = append(problems, fmt.Sprintf("%s: a grant's account is an account's name — letters, digits, "+ - "dots, underscores and dashes, not starting with a dash — and %q is not one", where, a.Grant.Account)) + if u.Use != AccessRead && u.Use != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where, + u.Identity, u.Use, AccessRead, AccessReadWrite)) + } + if u.Use == AccessReadWrite && a.Mode != AccessReadWrite { + problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where, + u.Identity, orRead(a.Mode))) + } + if u.From != FromContainer && u.From != FromToolRunner { + problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where, + u.Identity, u.From, FromContainer, FromToolRunner)) } } return problems } +// orRead is an access's mode as said: absent narrows to read. +func orRead(mode string) string { + if mode == "" { + return AccessRead + } + return mode +} + func (u *User) Identity() string { return u.ID } func (u *User) Kind() Type { return TypeUser } func (u *User) Target() string { return u.Name } @@ -1207,6 +1234,78 @@ type Container struct { // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. Health *Health `json:"health,omitempty"` + + // RunAs is who the container runs as, resolved by the controller from the module's identity and the + // image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and + // PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the + // container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because + // a resource's Identity is its id. + RunAs *ContainerIdentity `json:"identity,omitempty"` +} + +// The ways an image is told who to run as (novox/hq ADR 0325 §2). +const ( + IdentityByEnv = "env" + IdentityByRun = "run" +) + +// ContainerIdentity is who a container runs as, and how its image is told. +type ContainerIdentity struct { + // Mode is IdentityByEnv or IdentityByRun. + Mode string `json:"mode"` + // UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for + // IdentityByEnv: "PUID", "PGID". + UserEnv string `json:"user_env,omitempty"` + GroupEnv string `json:"group_env,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// UnmarshalJSON reads an identity as strictly as a resource is read. +func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error { + type plain ContainerIdentity + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("a container's identity: %w", err) + } + *i = ContainerIdentity(p) + return nil +} + +// envName is a variable's name as a runtime takes it. +var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (i *ContainerIdentity) problems(where string, env map[string]string) []string { + if i == nil { + return nil + } + var problems []string + switch i.Mode { + case IdentityByEnv: + if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) { + problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+ + "it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv)) + } + for _, name := range []string{i.UserEnv, i.GroupEnv} { + if _, set := env[name]; set && name != "" { + problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+ + "for one variable", where, name)) + } + } + case IdentityByRun: + if i.UserEnv != "" || i.GroupEnv != "" { + problems = append(problems, where+": an identity told by the runtime's user option names no variable") + } + default: + problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where, + IdentityByEnv, IdentityByRun, i.Mode)) + } + // Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started. + if i.UID <= 0 || i.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+ + "never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID)) + } + return problems } func (c *Container) Identity() string { return c.ID } @@ -1289,6 +1388,7 @@ func (c *Container) validate(where string, _ bool) []string { problems = append(problems, where+": networks is for a container that keeps running; a step "+ "runs and exits, and joins nothing afterwards") } + problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...) return append(problems, checkImage(where, c.Image)...) } @@ -1439,6 +1539,53 @@ type Declaration struct { // the generation of what it puts back, which may be older than what the machine applied, and is // not refused for it. PutBack bool + + // Identities is the identity of every module assigned to this machine (novox/hq ADR 0325 §3): an account + // and a group `mesh-` with the numbers the controller allocated once for the module, the same on + // every machine. The engine makes each before any resource is applied, and removes what it made of one no + // longer declared. Absent is a controller older than ADR 0325. + Identities []Identity +} + +// Identity is one module's identity, as the controller allocated it. +type Identity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +// IdentityPrefix starts every module identity's name. +const IdentityPrefix = "mesh-" + +// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number +// twice — two modules with one account would write as each other. +func checkIdentities(ids []Identity) []string { + var problems []string + names, uids, gids := map[string]bool{}, map[int]string{}, map[int]string{} + for _, id := range ids { + where := fmt.Sprintf("identity %q", id.Name) + if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) { + problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s, and this one is "+ + "for the module %q", where, IdentityPrefix, id.Module)) + } + if id.UID <= 0 || id.GID <= 0 { + problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+ + "nor a number below it (novox/hq ADR 0325 §11); nothing of this declaration was made", where, + id.UID, id.GID)) + } + if names[id.Name] { + problems = append(problems, where+": declared twice") + } + if other, used := uids[id.UID]; used { + problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other)) + } + if other, used := gids[id.GID]; used { + problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, id.GID, other)) + } + names[id.Name], uids[id.UID], gids[id.GID] = true, id.Name, id.Name + } + return problems } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1608,6 +1755,9 @@ type envelope struct { // sends them only to a host whose reports say `reads_generation`. Generation int64 `json:"generation,omitempty"` PutBack bool `json:"put_back,omitempty"` + // Identities is optional on the wire too (novox/hq ADR 0325 §3). **An older host refuses this key**; a + // controller sends it only to a host whose health says link.IdentityContract. + Identities []Identity `json:"identities,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1625,8 +1775,9 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} - var problems []string + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, + Identities: env.Identities} + problems := checkIdentities(env.Identities) if env.Generation < 0 { // Below zero would read as "none claimed" and pass every refusal of an older generation. problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ @@ -1715,7 +1866,6 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkWhileStopped(d.Resources)...) - problems = append(problems, checkGrants(d.Resources)...) problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) problems = append(problems, checkReplaces(d.Resources)...) if env.Adoption == nil { @@ -1780,36 +1930,10 @@ func checkWhileStopped(resources []Resource) []string { return problems } -// accountName is what a grant's account may be: the portable account name the user tools take, optionally ending -// in `$`, never starting with a dash. +// accountName is what an identity's or an access user's name may be: the portable account name the user tools +// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument. var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) -// checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it -// names is a container of this declaration, as a maintenance window's are. A grant to a container the machine -// is not told about would be a group given to nothing, said as granted. -func checkGrants(resources []Resource) []string { - containers := map[string]bool{} - for _, r := range resources { - if r.Kind() == TypeContainer { - containers[r.Identity()] = true - } - } - var problems []string - for _, r := range resources { - a, ok := r.(*Access) - if !ok || a.Grant == nil { - continue - } - for _, id := range a.Grant.Containers { - if id != "" && !containers[id] { - problems = append(problems, fmt.Sprintf("resource %q: its grant names %q, and this declaration "+ - "has no container by that id", a.ID, id)) - } - } - } - return problems -} - func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. diff --git a/internal/declaration/identity_test.go b/internal/declaration/identity_test.go new file mode 100644 index 0000000..e97225f --- /dev/null +++ b/internal/declaration/identity_test.go @@ -0,0 +1,61 @@ +package declaration + +import ( + "strings" + "testing" +) + +const anImage = "docker.io/library/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +// The fields ADR 0325 adds, read as the controller sends them (novox/hq ADR 0325 §2, §3, §5, issue 567). +func TestIdentitiesUsersAndARunAsAreReadAsSent(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1, + "identities":[{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}], + "resources":[ + {"id":"radarr.app","type":"container","name":"radarr","image":"` + anImage + `", + "identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":61001}}, + {"id":"plex.app","type":"container","name":"plex","image":"` + anImage + `","identity":{"mode":"run","uid":61002,"gid":61002}}, + {"id":"radarr.movies","type":"access","path":"/srv/media/movies","mode":"read-write","taken":true, + "users":[{"identity":"mesh-radarr","uid":61001,"use":"read-write","from":"container"}, + {"identity":"nox","uid":61003,"use":"read","from":"tool-runner"}]}, + {"id":"old.movies","type":"access","path":"/srv/media/movies","mode":"read-write","grant":{"account":"nox"}}]}`)) + if err != nil { + t.Fatal(err) + } + if len(d.Identities) != 1 || d.Identities[0].UID != 61001 { + t.Fatalf("identities: %+v", d.Identities) + } + if c := d.Resources[0].(*Container); c.RunAs == nil || c.RunAs.UserEnv != "PUID" { + t.Fatalf("a container's identity: %+v", c.RunAs) + } + if a := d.Resources[2].(*Access); !a.Taken || len(a.Users) != 2 || a.Users[1].From != FromToolRunner { + t.Fatalf("an access's users: %+v", a) + } +} + +// What the engine refuses on arrival: a root number, an identity not named for its module, a field it does not +// know, a user that writes a read access, a variable set twice. +func TestWhatADeclarationMayNotSayOfAnIdentity(t *testing.T) { + for want, raw := range map[string]string{ + "never root": `"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]`, + "named mesh-": `"identities":[{"module":"m","name":"m","uid":61001,"gid":61001}],` + + `"resources":[]`, + "is mesh-a's too": `"identities":[{"module":"a","name":"mesh-a","uid":61001,"gid":61001},` + + `{"module":"b","name":"mesh-b","uid":61001,"gid":61002}],"resources":[]`, + "unknown field": `"identities":[{"module":"m","name":"mesh-m","uid":61001,"gid":61001,"home":"/"}],` + + `"resources":[]`, + "writes an access declared read": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","mode":"read",` + + `"users":[{"identity":"mesh-m","uid":61001,"use":"read-write","from":"container"}]}]`, + "uid 0": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","users":[{"identity":"root","uid":0,` + + `"use":"read","from":"container"}]}]`, + "one value": `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"env":{"PUID":"1000"},"identity":{"mode":"env","user_env":"PUID","uid":61001,"gid":61001}}]`, + `"env" or "run"`: `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` + + `"identity":{"mode":"sudo","uid":61001,"gid":61001}}]`, + } { + _, err := Parse([]byte(`{"declaration":1,` + raw + `}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("want a refusal saying %q, got %v", want, err) + } + } +} diff --git a/internal/link/access_act_test.go b/internal/link/access_act_test.go new file mode 100644 index 0000000..3d745b2 --- /dev/null +++ b/internal/link/access_act_test.go @@ -0,0 +1,68 @@ +package link + +import ( + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" +) + +// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a +// signing context of its own. The controller's side holds the same names and words. +func TestTheTakeActsAskKeepsItsFieldNamesAndContext(t *testing.T) { + body, _ := json.Marshal(AccessActAsk{Act: ActTake, Access: "radarr.movies", Warrant: "w"}) + if got := keysOf(t, body); got != "access act warrant" { + t.Fatalf("the act's fields are %q", got) + } + if AskAccessActSubject("ace") != "mesh.node.ace.ask.access-act" { + t.Fatalf("the subject is %q", AskAccessActSubject("ace")) + } + if AccessActContext != "novox-mesh access-act v2\n" { + t.Fatalf("the signing context is %q", AccessActContext) + } +} + +// signedAct is an act's ask as the controller makes it: the hand-over's fields beside the act's, signed over the +// act's context and its bytes. +func signedAct(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, act AccessActAsk) []byte { + t.Helper() + raw, _ := json.Marshal(struct { + HandOverAsk + AccessActAsk + }{ask, act}) + body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(AccessActContext), raw...))}) + return body +} + +// **Nothing is done without a warrant**, nor for any act but take, nor signed as a hand-over (ADR 0325 §8). +func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) { + m, key := aMember(t) + now := time.Now() + var given []AccessAct + handle := func(a AccessAct) (string, error) { given = append(given, a); return "started", nil } + check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}} + ask := func(nonce string) HandOverAsk { + return HandOverAsk{Node: m.Node, Path: "/srv/media/movies", By: "the operator", Expires: now.Add(HandOverGood), + Nonce: nonce} + } + for why, body := range map[string][]byte{ + "no warrant": signedAct(t, key, ask(aNonce+"01"), AccessActAsk{Act: ActTake, Access: "radarr.movies"}), + "another act": signedAct(t, key, ask(aNonce+"02"), AccessActAsk{Act: "share", Access: "radarr.movies", Warrant: "w"}), + "no access": signedAct(t, key, ask(aNonce+"03"), AccessActAsk{Act: ActTake, Warrant: "w"}), + "a hand-over's": signedAsk(t, key, ask(aNonce+"04")), + "a path as access": signedAct(t, key, ask(aNonce+"05"), AccessActAsk{Act: ActTake, Access: "/srv", Warrant: "w"}), + } { + if a := AnswerAccessAct(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was changed") { + t.Errorf("%s: not refused, or refused without saying nothing was changed: %+v", why, a) + } + } + if len(given) != 0 { + t.Fatalf("a refused ask reached the act: %+v", given) + } + a := AnswerAccessAct(check, handle, signedAct(t, key, ask(aNonce+"06"), AccessActAsk{Act: ActTake, + Access: "radarr.movies", Warrant: "w-1"})) + if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" { + t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given) + } +} diff --git a/internal/link/handover.go b/internal/link/handover.go index 1d0dd2f..69eec73 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -136,20 +136,27 @@ func AnswerHandOver(check HandOverCheck, handle HandOverFunc, body []byte) HandO // one short line naming who asked, and a path when withPath and none otherwise. The nonce is taken last, after // every other check passed. It answers the ask with who asked trimmed, or why it is refused. func believed(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, string) { + ask, _, refused := believedBytes(check, context, withPath, body) + return ask, refused +} + +// believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's +// fields: read out of them only once they are believed. +func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) { if len(body) > MaxHandOverAsk { - return HandOverAsk{}, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body)) + return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body)) } var signed SignedHandOver if err := json.Unmarshal(body, &signed); err != nil { - return HandOverAsk{}, "the ask is not a signed ask: " + err.Error() + return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error() } if len(check.Signer) != ed25519.PublicKeySize || !ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) { - return HandOverAsk{}, "the ask is not signed by the mesh" + return HandOverAsk{}, nil, "the ask is not signed by the mesh" } var ask HandOverAsk if err := json.Unmarshal(signed.Ask, &ask); err != nil { - return HandOverAsk{}, "the signed ask is not an ask: " + err.Error() + return HandOverAsk{}, nil, "the signed ask is not an ask: " + err.Error() } now := time.Now() if check.Now != nil { @@ -163,25 +170,25 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) ( nonce, nonceErr := hex.DecodeString(ask.Nonce) switch { case check.Node == "" || ask.Node != check.Node: - return HandOverAsk{}, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node) + return HandOverAsk{}, nil, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node) case ask.Expires.IsZero() || !now.Before(ask.Expires): - return HandOverAsk{}, "the ask has expired" + return HandOverAsk{}, nil, "the ask has expired" case ask.Expires.After(now.Add(HandOverGood + HandOverSkew)): - return HandOverAsk{}, "the ask is good for longer than an ask may be" + return HandOverAsk{}, nil, "the ask is good for longer than an ask may be" case nonceErr != nil || len(nonce) < 16: - return HandOverAsk{}, "the ask carries no nonce of its own" + return HandOverAsk{}, nil, "the ask carries no nonce of its own" case ask.By == "": - return HandOverAsk{}, "the ask names nobody as asking" + return HandOverAsk{}, nil, "the ask names nobody as asking" case len(ask.By) > MaxHandOverBy || strings.ContainsAny(ask.By, "\r\n\x00"): - return HandOverAsk{}, "who asked is not one short line" + return HandOverAsk{}, nil, "who asked is not one short line" case withPath && ask.Path == "": - return HandOverAsk{}, "the ask names no directory" + return HandOverAsk{}, nil, "the ask names no directory" case !withPath && ask.Path != "": - return HandOverAsk{}, "the ask names a path, and this ask takes none" + return HandOverAsk{}, nil, "the ask names a path, and this ask takes none" case !nonces.take(ask.Nonce, ask.Expires, now): - return HandOverAsk{}, "the ask was taken already" + return HandOverAsk{}, nil, "the ask was taken already" } - return ask, "" + return ask, signed.Ask, "" } // HandOverReply is what a hand-over request is answered with, and whether to answer: a request with no reply @@ -320,3 +327,104 @@ func JudgeDirectoriesReply(check HandOverCheck, handle JudgeDirectoriesFunc, rep type DirectoryJudgements interface { HearDirectoryJudgements(JudgeDirectoriesFunc) } + +// The take act on an accessed folder, asked over the bus (novox/hq ADR 0325 §8, issue 567). +// +// A folder that already holds data is kept only on the operator's word: the access waits (§7), the controller asks +// the operator once for that folder on that machine (ADR 0259), and on the approved warrant asks this node's engine +// on AskAccessActSubject, signed as a hand-over is and held to the same checks (believed), under a signing context +// of its own, naming the act, the access, its folder as the engine states it, and the warrant. The engine judges +// every value against what it was last declared and what is on disk, answers at once that the act started or why +// it is refused, adds the users' entries to everything under the folder (following no link, entering no other +// filesystem), and states the act on the access's verdict (Act). Adding or dropping a user of a folder already +// taken asks nobody: the engine does it on the apply that declares it. + +// AskAccessActSubject is where the controller asks this node's engine to run an act on one access's folder. +func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" } + +// AccessActContext is prefixed to an act ask's bytes before signing: never another ask's, nor a declaration's. +const AccessActContext = "novox-mesh access-act v2\n" + +// The walks under an accessed folder. Only ActTake is asked; ActAdd and ActDrop are what an apply starts. +const ( + // ActTake adds the users' entries to the folder and everything under it, on a warrant (ADR 0325 §8). + ActTake = "take" + // ActAdd adds the entries of a user that joined a taken folder; ActDrop removes exactly a dropped user's. + ActAdd = "add" + ActDrop = "drop" +) + +// AccessActAsk is an act's own fields, beside a hand-over's (node, path, by, expires, nonce): which act, the +// access by its resource id, and the warrant the operator approved. +type AccessActAsk struct { + Act string `json:"act"` + Access string `json:"access"` + Warrant string `json:"warrant"` +} + +// AccessAct is one act asked and believed, as the handler is given it. +type AccessAct struct { + Act, Access, Path, Warrant, By string +} + +// MaxActField bounds an access id and a warrant id. +const MaxActField = 200 + +// AccessActFunc judges and runs one act: the words to answer with, or the refusal. +type AccessActFunc func(AccessAct) (string, error) + +func refusedAct(why string) HandOverAnswer { + return HandOverAnswer{Refused: why + "; nothing was changed"} +} + +// AnswerAccessAct is the engine's judgement of one act ask's bytes: believed as a hand-over is, then the act, the +// access and the warrant — **nothing is done without a warrant** — and only then the handler. +func AnswerAccessAct(check HandOverCheck, handle AccessActFunc, body []byte) HandOverAnswer { + if handle == nil { + return refusedAct("this engine runs no act on an accessed folder") + } + ask, signed, refused := believedBytes(check, AccessActContext, true, body) + if refused != "" { + return refusedAct(refused) + } + var act AccessActAsk + if err := json.Unmarshal(signed, &act); err != nil { + return refusedAct("the signed ask is not an act's ask: " + err.Error()) + } + act.Access, act.Warrant = strings.TrimSpace(act.Access), strings.TrimSpace(act.Warrant) + switch { + case act.Warrant == "": + return refusedAct("the ask names no warrant, and an act on an accessed folder runs only on one") + case act.Access == "": + return refusedAct("the ask names no access") + case act.Act != ActTake: + return refusedAct(fmt.Sprintf("the ask names the act %q, and the one act asked is %q", act.Act, ActTake)) + } + for _, v := range []string{act.Access, act.Warrant} { + if len(v) > MaxActField || strings.ContainsAny(v, "\r\n\x00/") { + return refusedAct("the ask's access or warrant is not one short name") + } + } + said, err := handle(AccessAct{Act: act.Act, Access: act.Access, Path: ask.Path, Warrant: act.Warrant, By: ask.By}) + if err != nil { + return refusedAct(err.Error()) + } + return HandOverAnswer{Said: said} +} + +// AccessActReply is what an act request is answered with, and whether to answer, as HandOverReply. +func AccessActReply(check HandOverCheck, handle AccessActFunc, reply string, body []byte) ([]byte, bool) { + if reply == "" { + return nil, false + } + answer, err := json.Marshal(AnswerAccessAct(check, handle, body)) + if err != nil { + return nil, false + } + return answer, true +} + +// AccessActs is a link that hears the controller asking this node to run an act on an accessed folder. Optional. +type AccessActs interface { + HearAccessActs(AccessActFunc) +} diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index fd27d0f..d4bebc4 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -66,7 +66,9 @@ type natsLink struct { searching *nats.Subscription // judging hears the controller asking for a read-only judgement of this node's directories (issue 358). judging *nats.Subscription - node string + // acting hears the controller asking for the take act on an access's folder (novox/hq ADR 0325 §8). + acting *nats.Subscription + node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). signer ed25519.PublicKey arrived chan Declaration @@ -259,6 +261,24 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) { } } +// HearAccessActs listens for the controller asking this node to run the take act on an access's folder (novox/hq ADR +// 0325 §8), on this machine's own subject, verified as a hand-over is (AnswerAccessAct). A bus whose user list is older +// than the grant refuses the subscription, said by the client; the controller then hears no answer. +func (l *natsLink) HearAccessActs(handle AccessActFunc) { + if handle == nil { + return + } + check := HandOverCheck{Signer: l.signer, Node: l.node} + acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) { + if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer { + _ = msg.Respond(body) + } + }) + if err == nil { + l.acting = acting + } +} + func (l *natsLink) Declarations() <-chan Declaration { return l.arrived } func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } func (l *natsLink) Lost() <-chan error { return l.lost } @@ -276,6 +296,9 @@ func (l *natsLink) Close() { if l.searching != nil { _ = l.searching.Unsubscribe() } + if l.acting != nil { + _ = l.acting.Unsubscribe() + } if l.judging != nil { _ = l.judging.Unsubscribe() } diff --git a/internal/link/messages.go b/internal/link/messages.go index 3f84232..88083ca 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -258,6 +258,14 @@ const RootContract = 3 // know. const GrantContract = 4 +// IdentityContract is the statement of a host that reads a declaration's `identities`, a container's `identity` and +// an access's `users` and `taken` (novox/hq ADR 0325): it makes each module's account, runs a container as its +// module's identity, keeps each shared folder usable through ACL entries, states an access it may not keep yet as +// waiting with a wait of kind access, and hears the take act. It no longer grants an access's group (ADR 0312 is +// superseded): a `grant` still sent is accepted and not acted on. Like the fields before it, its presence is what +// tells the controller this host may be sent those fields; an older host refuses the whole declaration for them. +const IdentityContract = 5 + // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. @@ -296,9 +304,9 @@ const KindAccount = "account" // starting ReasonUsedAsFound, until the operator hands it to the mesh (`nox node hand-over ` on the control-node, novox/hq issue 356). const KindDirectory = "directory" -// KindAccess is a read-write access that grants its path's group (novox/hq ADR 0312): healthy while the group may -// write the path, unhealthy while it cannot — the operator's to decide, since the mesh changes nothing on the path -// (§2) — or while the path is owned by a group that is never granted (§4). +// KindAccess is an access whose folder the engine keeps for its users (novox/hq ADR 0325): healthy while every user +// has its effective right on the folder and the sweep's sample under it (§10), waiting while the mesh may not keep +// it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12). const KindAccess = "access" // ReasonUsedAsFound starts the reason of a directory used as found. @@ -413,6 +421,75 @@ type ResourceHealth struct { // Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one // setting of the module by name, the part, and what the operator gives, in words. Absent otherwise. Waits []health.Wait `json:"waits,omitempty"` + // AccessWaits is what an access stated waiting waits for (novox/hq ADR 0325 §7). Beside Waits rather than in + // it because mesh-sdk's health.Wait names a secret or a setting and nothing else; it moves there once that type + // has a kind. Absent otherwise. + AccessWaits []AccessWait `json:"access-waits,omitempty"` + // Act is the last walk under an access's folder (ADR 0325 §8): a take on a warrant, or an add or a drop of a + // user's entries. Absent on every other verdict, and on a folder never walked. + Act *AccessActRecord `json:"act,omitempty"` + // Entries is the listing of a kept folder's users, for an access's verdict (ADR 0325 §4): each identity, its + // use, and a note beside the tool runner's entry. Absent on a folder not kept. + Entries []KeptEntry `json:"entries,omitempty"` +} + +// WaitKindAccess is the kind of a wait on an access (novox/hq ADR 0325 §7). +const WaitKindAccess = "access" + +// AccessWait is one access waiting for the operator: the access by its module's own id for it, the part (its +// module), and what the operator gives, in words with no path (ADR 0234 §6): "let the mesh keep the shared folder +// movies", or "mount the disk that holds books". +type AccessWait struct { + // Kind is WaitKindAccess. + Kind string `json:"kind"` + Access string `json:"access"` + Part string `json:"part"` + What string `json:"what"` +} + +// KeptEntry is one user of a kept folder as its listing says it. +type KeptEntry struct { + Identity string `json:"identity"` + UID int `json:"uid"` + Use string `json:"use"` + From string `json:"from"` + // Note is said beside the tool runner's entry (ADR 0325 §4): it stands for every module whose tools use the + // folder, at the widest use any of them declares. + Note string `json:"note,omitempty"` +} + +// The states of a walk under an accessed folder. +const ( + ActRunning = "running" + ActDone = "done" + // ActStopped is a walk that met its bound in entries or time: what it changed stays, and what it did not + // reach is not changed. Asked again, or on the next apply, it goes on. + ActStopped = "stopped" + ActFailed = "failed" +) + +// AccessActRecord is one walk under an access's folder, as the engine records it (novox/hq ADR 0325 §8). Counts +// only: never a path. +type AccessActRecord struct { + // Act is ActTake, ActAdd or ActDrop. + Act string `json:"act"` + Access string `json:"access"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + State string `json:"state"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended,omitzero"` + // Folders and Files are how many it changed; Kept how many already had their entries; Links the links it + // passed by unfollowed; Other the entries it left that are neither a file nor a folder; Mounts the folders on + // another filesystem it did not enter. + Folders int `json:"folders,omitempty"` + Files int `json:"files,omitempty"` + Kept int `json:"kept,omitempty"` + Links int `json:"links,omitempty"` + Other int `json:"other,omitempty"` + Mounts int `json:"mounts,omitempty"` + // Why is what stopped or failed it, in words with no path. + Why string `json:"why,omitempty"` } // SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times diff --git a/internal/link/queue.go b/internal/link/queue.go index 227dc1d..f24cdb1 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -62,8 +62,11 @@ type Queue struct { // JudgeDirectories judges this node's declared directories read-only, asked by the controller over the // link (novox/hq issue 358). Nil answers every such ask with a refusal. JudgeDirectories JudgeDirectoriesFunc - Say Announce - Timeout time.Duration + // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved + // warrant (novox/hq ADR 0325 §8). Nil answers every such ask with a refusal. + AccessAct AccessActFunc + Say Announce + Timeout time.Duration once sync.Once wake chan struct{} diff --git a/internal/link/run.go b/internal/link/run.go index b61782e..f45d808 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -252,6 +252,10 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou if j, ok := link.(DirectoryJudgements); ok { j.HearDirectoryJudgements(queue.JudgeDirectories) } + // And asking for the take act on an access's folder, on a warrant (novox/hq ADR 0325 §8). + if s, ok := link.(AccessActs); ok { + s.HearAccessActs(queue.AccessAct) + } for { // Asked to stop — by standing aside, say — nothing further is taken in. diff --git a/internal/store/kept.go b/internal/store/kept.go new file mode 100644 index 0000000..ed63fbc --- /dev/null +++ b/internal/store/kept.go @@ -0,0 +1,183 @@ +package store + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" +) + +// KeptName is the file, beside the node's state, of every shared folder the engine keeps (novox/hq ADR 0325 §5): +// whose entries it added, on which filesystem it was placed, whether it was taken and under which warrant, and how +// far its sweep got. Its own file for the reason the hand-overs have theirs: the take act, the watch and the sweep +// run beside an apply, whose save of the state would lose a record written into it meanwhile. +const KeptName = "kept-folders.json" + +// KeptPath is that file beside the state at statePath. +func KeptPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), KeptName) +} + +// KeptUser is one identity the mesh keeps an entry for on a folder: its name, its number, its use and where it +// runs (a container, or the tool runner standing for every module whose tools use the folder, ADR 0325 §4). +type KeptUser struct { + Identity string `json:"identity"` + UID uint32 `json:"uid"` + Use string `json:"use"` + From string `json:"from"` +} + +// How a folder came to be kept. +const ( + // KeptByWarrant is a folder that held data, taken on the operator's word (ADR 0325 §8). + KeptByWarrant = "warrant" + // KeptEmpty is a folder that held nothing yet when it was first kept (§5). + KeptEmpty = "empty" + // KeptMade is a folder the engine made under a parent on its placed filesystem (§9). + KeptMade = "made" +) + +// KeptFolder is one shared folder the engine keeps, or has seen and waits to keep. Counts and mount points only +// beside its own path: never a path under it. +type KeptFolder struct { + Path string `json:"path"` + // Mount is the mount point of the filesystem the folder was placed on when the engine first saw it there: + // later, a nearest existing parent on another one is the disk not there (ADR 0325 §9). + Mount string `json:"mount,omitempty"` + // Kept is that the mesh keeps it; How why (KeptByWarrant, KeptEmpty, KeptMade); Warrant and By the warrant and + // who asked, for a folder taken on one; At since when. + Kept bool `json:"kept,omitempty"` + How string `json:"how,omitempty"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at,omitzero"` + // Users are the identities whose entries are on everything under the folder; Want what the last apply + // declared. Where they differ a walk adds or removes exactly the difference. + Users []KeptUser `json:"users,omitempty"` + Want []KeptUser `json:"want,omitempty"` + // Act is the last walk under the folder: a take, an add or a drop. + Act *KeptAct `json:"act,omitempty"` + // Sweep is how far the sweep got (ADR 0325 §10). + Sweep KeptSweep `json:"sweep"` + // Watches is how many folders under it are watched; OutOfWatches that the machine's watch limit was met, and + // the sweep alone finds what is made where no watch is (§6). + Watches int `json:"watches,omitempty"` + OutOfWatches bool `json:"out_of_watches,omitempty"` +} + +// KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. +type KeptAct struct { + // Act is take, add or drop. + Act string `json:"act"` + Warrant string `json:"warrant,omitempty"` + By string `json:"by,omitempty"` + State string `json:"state"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended,omitzero"` + // Folders and Files are the entries changed; Kept the ones that already had what they need; Links the links + // passed by unfollowed; Other the entries neither a file nor a folder; Mounts the folders on another + // filesystem not entered. + Folders int `json:"folders,omitempty"` + Files int `json:"files,omitempty"` + Kept int `json:"kept,omitempty"` + Links int `json:"links,omitempty"` + Other int `json:"other,omitempty"` + Mounts int `json:"mounts,omitempty"` + // Why is what stopped or failed it, in words with no path. + Why string `json:"why,omitempty"` +} + +// KeptSweep is the sweep's place in a kept folder: where the last look stopped, below the folder, by the names +// of each level (never said, only resumed from); when it last covered the folder whole; and what this pass found. +type KeptSweep struct { + Cursor []string `json:"cursor,omitempty"` + Covered time.Time `json:"covered,omitzero"` + // Read is how many entries this pass read so far; Narrowed how many gave a user less than its right and were + // put back; Unrepaired how many could not be (an owner's bits, which the mesh never changes, or a write that + // failed). + Read int `json:"read,omitempty"` + Narrowed int `json:"narrowed,omitempty"` + Unrepaired int `json:"unrepaired,omitempty"` + // LastUnrepaired is what the last whole pass could not put back: said until the next pass is through. + LastUnrepaired int `json:"last_unrepaired,omitempty"` + // Bound is that the last look ran out of its bound before the folder was covered. + Bound bool `json:"bound,omitempty"` +} + +// KeptFile is the kept folders' file, one writer at a time in this process. +type KeptFile struct { + path string + mu sync.Mutex +} + +// NewKeptFile is the kept folders' file beside the state at statePath. +func NewKeptFile(statePath string) *KeptFile { return &KeptFile{path: KeptPath(statePath)} } + +// All is every kept or seen folder, by path. None is an empty map; a file that cannot be read is an error. +func (k *KeptFile) All() (map[string]KeptFolder, error) { + k.mu.Lock() + defer k.mu.Unlock() + return k.load() +} + +// Update changes the record of one folder under the file's lock: change is given the record (empty when there is +// none) and whether there was one, and says whether to keep it; a record not kept is removed. +func (k *KeptFile) Update(path string, change func(f *KeptFolder, had bool) bool) error { + k.mu.Lock() + defer k.mu.Unlock() + all, err := k.load() + if err != nil { + return err + } + f, had := all[path] + f.Path = path + if change(&f, had) { + all[path] = f + } else { + delete(all, path) + } + return k.save(all) +} + +func (k *KeptFile) load() (map[string]KeptFolder, error) { + raw, err := os.ReadFile(k.path) + if errors.Is(err, fs.ErrNotExist) { + return map[string]KeptFolder{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading the kept folders: %w", err) + } + out := map[string]KeptFolder{} + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the kept folders (%s) are unreadable: %w", k.path, err) + } + return out, nil +} + +func (k *KeptFile) save(all map[string]KeptFolder) error { + raw, err := json.MarshalIndent(all, "", " ") + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(k.path), ".kept-folders-*.json") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(append(raw, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), k.path) +} diff --git a/internal/store/store.go b/internal/store/store.go index 44ad307..b1bba24 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -241,6 +241,23 @@ type State struct { // network stays declared: undeclaring brings nothing back (ADR 0118), and a private network // assigned again finds the tunnel's configuration retired rather than missing. Retired []Retired `json:"retired,omitempty"` + + // Identities is every module identity the engine made here (novox/hq ADR 0325 §3): its account and group, + // by name and number, and which of the two it made. An identity no longer declared is removed, and only + // what the engine made: an account or group that was there before is never the mesh's to remove. + Identities []MadeIdentity `json:"identities,omitempty"` +} + +// MadeIdentity is one module identity as the engine made it. +type MadeIdentity struct { + Module string `json:"module"` + Name string `json:"name"` + UID int `json:"uid"` + GID int `json:"gid"` + // MadeUser and MadeGroup say which the engine made: only those are removed. + MadeUser bool `json:"made_user,omitempty"` + MadeGroup bool `json:"made_group,omitempty"` + At time.Time `json:"at"` } // Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq diff --git a/internal/system/identities.go b/internal/system/identities.go new file mode 100644 index 0000000..1c14265 --- /dev/null +++ b/internal/system/identities.go @@ -0,0 +1,87 @@ +package system + +import ( + "context" + "fmt" + "strconv" + "strings" +) + +// A module's identity as a system account (novox/hq ADR 0325 §3): an account and a group with exactly the numbers +// the controller allocated, no login shell, no home and no password. Optional beside System: a system that has +// none (android) makes no module identity, and the engine says so. + +// CreateSystemGroup makes a system group with gid. +func (arch) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error { + if _, err := run(ctx, "groupadd", "--system", "--gid", strconv.Itoa(gid), name); err != nil { + return fmt.Errorf("cannot create the group %q: %w", name, err) + } + return nil +} + +// CreateSystemAccount makes a system account with uid and the group gid as its own: no home made, `/` as the +// home field, nologin as its shell, and the password useradd leaves locked. +func (arch) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error { + if _, err := run(ctx, "useradd", "--system", "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(gid), + "--no-create-home", "--home-dir", "/", "--shell", "/usr/bin/nologin", name); err != nil { + return fmt.Errorf("cannot create the account %q: %w", name, err) + } + return nil +} + +// RemoveAccount removes an account, leaving every file it owns where it is. +func (arch) RemoveAccount(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "userdel", name); err != nil { + return fmt.Errorf("cannot remove the account %q: %w", name, err) + } + return nil +} + +// RemoveGroup removes a group. +func (arch) RemoveGroup(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "groupdel", name); err != nil { + return fmt.Errorf("cannot remove the group %q: %w", name, err) + } + return nil +} + +// CreateSystemGroup makes a system group with gid, with busybox's addgroup. +func (alpine) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error { + if _, err := run(ctx, "addgroup", "-S", "-g", strconv.Itoa(gid), name); err != nil { + return fmt.Errorf("cannot create the group %q: %w", name, err) + } + return nil +} + +// CreateSystemAccount makes a system account with busybox's adduser: -S system, -D no password, -H no home made. +func (alpine) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error { + group, err := run(ctx, "getent", "group", strconv.Itoa(gid)) + if err != nil { + return fmt.Errorf("cannot create the account %q: its group gid %d is not here: %w", name, gid, err) + } + groupName := name + if n, _, ok := strings.Cut(group, ":"); ok { + groupName = n + } + if _, err := run(ctx, "adduser", "-S", "-D", "-H", "-h", "/", "-s", "/sbin/nologin", "-u", strconv.Itoa(uid), + "-G", groupName, name); err != nil { + return fmt.Errorf("cannot create the account %q: %w", name, err) + } + return nil +} + +// RemoveAccount removes an account with busybox's deluser. +func (alpine) RemoveAccount(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "deluser", name); err != nil { + return fmt.Errorf("cannot remove the account %q: %w", name, err) + } + return nil +} + +// RemoveGroup removes a group with busybox's delgroup. +func (alpine) RemoveGroup(ctx context.Context, run Runner, name string) error { + if _, err := run(ctx, "delgroup", name); err != nil { + return fmt.Errorf("cannot remove the group %q: %w", name, err) + } + return nil +} From 2026ef36f383b7cfe1168c523057987e5e239a36 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:55:47 +0200 Subject: [PATCH 02/10] Judge a kept folder before every walk, and keep nothing a disk mounted over (hq ADR 0325 review) A walk run while a dataset was unmounted wrote onto root's empty mount point, or "dropped" entries from it and forgot the ones on the data; a folder kept while empty below an unnamed mount was then swept whole once the disk mounted, with no warrant. Every walk is now judged as an apply is, a changed mount judges the folder afresh, and systemd .mount units and mounts seen are read for the first-sight rule. Widening a mask caps the entries it limited, so the owning group gains nothing. The watch lets go of removed folders, re-arms on lost notice, reads users under its lock and ends with its context. A drop that cannot read an ACL is counted, a broken folder's words carry no path, a grant's group that could not be left is tried again, and a folder's stat is bounded. --- cmd/mesh-host/kept.go | 5 +- internal/apply/acl.go | 16 +- internal/apply/apply.go | 9 +- internal/apply/identities_test.go | 36 ++++ internal/apply/keep.go | 149 ++++++++++++---- internal/apply/keeper_linux.go | 30 +++- internal/apply/kept_test.go | 280 +++++++++++++++++++++++++++++- internal/apply/placed.go | 82 +++++++-- internal/apply/walk.go | 5 +- internal/apply/walk_linux.go | 10 +- internal/apply/watch_linux.go | 94 ++++++++-- 11 files changed, 641 insertions(+), 75 deletions(-) diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index 0a4e36e..ba742b5 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -162,7 +162,8 @@ func judgeAccess(a judgedAccess) verdict { case apply.VerdictRoot: return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)} case apply.VerdictBroken: - return verdict{state: link.StateUnhealthy, reason: strings.ReplaceAll(v.Why, a.path, "the folder")} + // Its words carry no path (AccessVerdict says them so): the reason reaches the operator's channel. + return verdict{state: link.StateUnhealthy, reason: v.Why} case apply.VerdictMake: return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " + "its data lives (novox/hq ADR 0325 §9)"} @@ -183,7 +184,7 @@ func judgeAccess(a judgedAccess) verdict { look := apply.LookKept(a.path, a.users, apply.SweepLimits) var faults []string if look.Failed != "" { - faults = append(faults, "the sweep could not look: "+strings.ReplaceAll(look.Failed, a.path, "the folder")) + faults = append(faults, "the sweep could not look: "+apply.Pathless(errors.New(look.Failed))) } if len(look.Short) > 0 { faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+ diff --git a/internal/apply/acl.go b/internal/apply/acl.go index 8874ec4..6fbab09 100644 --- a/internal/apply/acl.go +++ b/internal/apply/acl.go @@ -142,7 +142,7 @@ func (a acl) check() error { // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps -// what it gave every other entry; with none yet (a file the mode alone described) the mask starts at the owning +// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. // Nothing else is changed. func keptACL(a acl, need map[uint32]uint16) acl { @@ -157,7 +157,21 @@ func keptACL(a acl, need map[uint32]uint16) acl { } } if i := out.find(aclMask, 0); i >= 0 { + old := out[i].Perm out[i].Perm |= widen + if out[i].Perm != old { + // **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does + // not set here were limited by the old mask: a file made at 0644 under a default ACL carries the + // owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its + // effective right stays exactly as it was. + for j := range out { + e := &out[j] + _, ours := need[e.ID] + if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) { + e.Perm &= old + } + } + } return out } if len(need) == 0 { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index c5f6028..31e4aaa 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -353,12 +353,19 @@ func ApplyMindingWindows( // it (novox/hq ADR 0325 §8), never by removing the access's record. action, detail = "forgotten", "an operator-owned folder is never the host's to remove; the entries "+ "the mesh added go once no access names it" - if gave, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed); said != "" { + gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed) + if said != "" { detail = joinDetail(detail, said) if gave { action = "restored" } } + if len(left) > 0 { + // Kept recorded with what is still to take back, and tried again (ADR 0177's waiting removal). + orphan.Groups = left + known.Record(orphan) + err = fmt.Errorf("%w: %s", errRemovalWaits, said) + } default: action, detail, err = remove(ctx, sys, orphan, run, made) } diff --git a/internal/apply/identities_test.go b/internal/apply/identities_test.go index a31dd09..023e088 100644 --- a/internal/apply/identities_test.go +++ b/internal/apply/identities_test.go @@ -249,3 +249,39 @@ func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) { t.Errorf("the record still holds the grant's groups: %+v", rec) } } + +// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again. +func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) { + dir := t.TempDir() + refuse := true + groups := "nox media" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch name { + case "id": + return groups + "\n", nil + case "gpasswd": + if refuse { + return "", errors.New("gpasswd exited 1: locked") + } + groups = "nox" + } + return "", nil + } + known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir, + Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}} + d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`) + _, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 { + t.Fatalf("a group that could not be left was forgotten: %+v", rec) + } + refuse = false + if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil { + t.Fatal(err) + } + if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" { + t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups) + } +} diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 000dc1e..440f5eb 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -198,21 +199,23 @@ type folderVerdict struct { // why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link). why string err error + // moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh. + moved bool } -// assessFolder judges a folder as an apply and a look both do (ADR 0325 §7, §9, §12): its disk first, then whose -// it is, then whether the mesh may keep it. +// assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then +// whose it is, then whether the mesh may keep it. Its words carry no path: they reach the operator's channel. func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict { others := recordedMounts(path) - info, err := os.Lstat(path) + info, err := lstatWithin(path) if errors.Is(err, os.ErrNotExist) { m, err := NearestAncestor(path) if err != nil { return folderVerdict{err: err} } if !m.Info.IsDir() { - return folderVerdict{err: fmt.Errorf("%s is not there, and its nearest existing parent %s is not a "+ - "folder (a file or a link); nothing is made under it", path, m.Ancestor)} + return folderVerdict{err: errors.New("the folder is not there, and its nearest existing parent is not " + + "a folder (a file or a link); nothing is made under it")} } pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others) if pl.Away { @@ -220,19 +223,20 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } uid, gid, ok := statOwner(m.Info) if !ok { - return folderVerdict{err: fmt.Errorf("the owner of %s cannot be read on this machine", m.Ancestor)} + return folderVerdict{err: errors.New("the owner of the folder's nearest existing parent cannot be read " + + "on this machine")} } if why := rootOwned(uint32(uid), uint32(gid)); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " + "existing parent is " + why} } - return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade} + return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade, moved: pl.Moved} } if err != nil { - return folderVerdict{err: err} + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} } if !info.IsDir() { - return folderVerdict{err: fmt.Errorf("%s is not a directory, and an access is a shared directory", path)} + return folderVerdict{err: errors.New("the folder is not a directory, and an access is a shared directory")} } pl := placedOn(ctx, path, path, rec.Mount, others) if pl.Away { @@ -240,20 +244,66 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } facts, err := readFolder(path) if err != nil { - return folderVerdict{err: err} + return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} } if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } switch { - case rec.Kept: + case rec.Kept && !pl.Moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} case taken: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved} case facts.Empty: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved} } - return folderVerdict{kind: folderWaitKeep, mount: pl.Mount} + // What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty + // below it included (a move): it waits for the operator's word. + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved} +} + +// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose +// server is down blocks a stat for ever, and one access must never hold the apply. +var statBound = 5 * time.Second + +// errNoAnswer is a folder whose read did not return within statBound. +var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)") + +// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own. +func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) } +func statWithin(path string) (os.FileInfo, error) { return within(os.Stat, path) } + +func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + type read struct { + info os.FileInfo + err error + } + done := make(chan read, 1) + go func() { + info, err := stat(path) + done <- read{info, err} + }() + select { + case r := <-done: + return r.info, r.err + case <-time.After(statBound): + return nil, errNoAnswer + } +} + +// absPath is an absolute path in words: what pathless takes out. +var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`) + +// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other +// path said as "a path". +func Pathless(err error) string { return pathless(err) } + +func pathless(err error) string { + var pe *os.PathError + if errors.As(err, &pe) { + return pe.Op + ": " + pe.Err.Error() + } + return absPath.ReplaceAllString(err.Error(), "${1}a path") } // recordedMounts is every mount point recorded for another kept folder: where a disk was once seen. @@ -283,8 +333,11 @@ func RootWords(why string) string { func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied, in inputs) (Outcome, error) { // What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5). - gave, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) + gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed) out, err := applyAccessFolder(ctx, r, in) + if len(left) > 0 { + out.user, out.groups = previous.User, left + } if said != "" { out.Detail = joinDetail(out.Detail, said) if gave && out.Action == "unchanged" { @@ -307,9 +360,20 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O if v.err != nil { return out, v.err } + if v.moved { + // What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty + // below it): judged afresh, the record of what was kept there no longer holds. + _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { + *f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act} + return had + }) + } note := "" + if v.moved { + note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh" + } if len(r.Grant) > 0 { - note = "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" + note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" } switch v.kind { case folderWaitDisk: @@ -410,16 +474,41 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O // takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes // back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group. -// Whether any was taken back, and what was said; nothing for an access that granted nothing. +// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next +// apply tries again — and what was said; nothing for an access that granted nothing. func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted, - handed Handed) (bool, string) { + handed Handed) (bool, []string, string) { if previous.User == "" || len(previous.Groups) == 0 { - return false, "" + return false, nil, "" } - account := previous - account.Target = previous.User - gave, said := giveGroupsBack(ctx, sys, account, run, wanted, handed) - return gave, "the group grant ADR 0312 made is taken back: " + previous.User + " " + said + account := previous.User + in, err := system.GroupsOf(ctx, system.Runner(run), account) + if err != nil { + return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+ + "back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err) + } + already := setOf(in) + gave := false + var left, said []string + for _, g := range previous.Groups { + switch { + case !already[g]: + continue + case len(wanted.othersAsk(account, g, previous.ID)) > 0: + said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g, + strings.Join(wanted.othersAsk(account, g, previous.ID), ", "))) + handed.hand(account, g) + continue + } + gone, why := leaveGroup(ctx, sys, account, g, run) + if !gone { + left = append(left, g) + why += "; tried again on the next apply" + } + gave = gave || gone + said = append(said, why) + } + return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ") } // usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)". @@ -469,23 +558,23 @@ func NearestAncestor(path string) (MissingPath, error) { if err := keptPath(path); err != nil { return MissingPath{}, err } - if _, err := os.Lstat(path); err == nil || !errors.Is(err, os.ErrNotExist) { - return MissingPath{}, fmt.Errorf("%s is there, or cannot be read: %v", path, err) + if _, err := lstatWithin(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return MissingPath{}, errors.New("the folder is there, or cannot be read") } var levels []string for p := path; p != "/"; p = filepath.Dir(p) { levels = append([]string{filepath.Base(p)}, levels...) parent := filepath.Dir(p) - info, err := os.Lstat(parent) + info, err := lstatWithin(parent) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { - return MissingPath{}, err + return MissingPath{}, errors.New("a parent of the folder cannot be read: " + pathless(err)) } return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil } - return MissingPath{}, errors.New("no part of the path is there") + return MissingPath{}, errors.New("no part of the folder's path is there") } // accessPaths is every declared access's folder with its id: what no container mounts while it is not there. @@ -511,7 +600,7 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string if !atOrBelow(filepath.Clean(src), path) { continue } - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) { return id } } @@ -563,7 +652,7 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud v := assessFolder(ctx, path, rec, taken) switch { case v.err != nil: - return AccessJudgement{Kind: VerdictBroken, Why: v.err.Error()} + return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)} case v.kind == folderWaitDisk: return AccessJudgement{Kind: VerdictWaitDisk} case !used: diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index e87498a..5faeb71 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -64,15 +64,36 @@ func ReconcileKept(ctx context.Context) []string { if !f.Kept || sameUsers(f.Users, f.Want) { continue } - said = append(said, reconcileOne(path, f)...) + said = append(said, reconcileOne(ctx, path, f)...) } return said } -func reconcileOne(path string, f store.KeptFolder) []string { +func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string { lock := folderLock(path) lock.Lock() defer lock.Unlock() + // **Judged as an apply judges it, before every walk** (§9, §12): a disk not there leaves its mount point — + // an empty folder of root's on the filesystem below — at the path, and a walk there would write onto root's + // folder, or "drop" entries from a folder that never had them and forget the ones on the data. Nothing is + // walked then, and the record stays for the walk that can reach the data. + if v := assessFolder(ctx, path, f, false); v.err != nil || v.kind != folderKeep || v.moved { + why := "the disk that holds it is not there" + switch { + case v.err != nil: + why = pathless(v.err) + case v.kind == folderRoot: + why = RootWords(v.why) + case v.moved || v.kind == folderWaitKeep: + why = "what is at it now is not what the mesh kept there" + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Act = &store.KeptAct{Act: "walk", State: "waiting", Started: time.Now().UTC(), + Why: "not walked: " + why + "; walked once it is"} + return had + }) + return []string{fmt.Sprintf("the kept folder %s: not walked, %s", path, why)} + } have, want := usersOf(f.Users), usersOf(f.Want) drop := map[uint32]bool{} adds := false @@ -98,7 +119,8 @@ func reconcileOne(path string, f store.KeptFolder) []string { rec := actRecord(act, "", "", started, w, err) _ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool { k.Act = rec - if err == nil { + // Done only when every entry was reached and changed: one that failed is walked again. + if err == nil && w.Failed == "" { k.Users = f.Want } return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0) @@ -174,7 +196,7 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } started := time.Now().UTC() _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - if !f.Kept { + if !f.Kept || v.moved { f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil } f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 10f97dd..57a7efa 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -56,9 +56,13 @@ func keeping(t *testing.T, points []string, targets []string) { t.Helper() was, wasPoints, wasTargets := KeptFolders, MountPoints, MountTargets KeptFolders = NewKeptMemory() + forgetMountHistory() MountPoints = func() []string { return append([]string{"/"}, points...) } MountTargets = func(context.Context) []string { return targets } - t.Cleanup(func() { KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets }) + t.Cleanup(func() { + KeptFolders, MountPoints, MountTargets = was, wasPoints, wasTargets + forgetMountHistory() + }) } // accessJSON is an access of module to path with users, each "identity:uid:use:from". @@ -227,8 +231,9 @@ func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { t.Skipf("no inotify here: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go w.Run(ctx) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + defer func() { cancel(); <-ended }() w.Set(map[string][]store.KeptUser{lib: rec.Want}) narrow := filepath.Join(lib, "narrow") @@ -587,3 +592,272 @@ func TestAnACLReadsBackAsWritten(t *testing.T) { t.Errorf("mask %s, and the entries need rw- and the group had r--", permWords(m)) } } + +func forgetMountHistory() { + mountHistory.Lock() + mountHistory.seen = map[string]bool{} + mountHistory.Unlock() +} + +// Review H1: a walk is judged as an apply is, first. With the dataset not mounted its mount point is what is at the +// path, and neither an add nor a drop walk touches it; the record stays for the walk that can reach the data. +func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) { + dir := aclFolder(t) + dataset := filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib := filepath.Join(dataset, "movies") + both := []string{"mesh-radarr:61001:read-write:container", "mesh-plex:61002:read:container"} + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both...)); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); rec.Mount != dataset || !sameUsers(rec.Users, rec.Want) { + t.Fatalf("not kept on its dataset: %+v", rec) + } + // plex drops it, and the dataset is unmounted before the walk. + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + both[0])); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + rec, had := keptRecord(lib) + if !had || len(rec.Users) != 2 { + t.Fatalf("a drop that could not reach the data forgot whose entries are on it: %+v", rec) + } + if rec.Act == nil || rec.Act.State != "waiting" { + t.Errorf("the walk not run is not said: %+v", rec.Act) + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); !still { + t.Fatal("a walk ran where the disk is not there") + } + // The disk back: the drop runs. + MountPoints = func() []string { return []string{"/", dataset} } + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); still { + t.Fatal("the drop did not run once the disk was there") + } +} + +// Review H2: a folder kept while it was empty on the filesystem below a mount nobody named is judged afresh when +// the disk mounts over it: holding data, it waits for the operator's word, and nothing is added to it. +func TestADiskMountedOverAFolderKeptEmptyIsNotKeptWithoutAWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if rec, _ := keptRecord(lib); !rec.Kept || rec.How != store.KeptEmpty || rec.Mount != "/" { + t.Fatalf("not kept as empty on /: %+v", rec) + } + // The pool is imported: a library is at the path now, on its own filesystem. + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + // The new disk's data carries no entries of the mesh's: the test's stand-in made it under the folder kept + // below, so what it inherited there is taken off. + if err := unix.Removexattr(film, aclAccessXattr); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/", pool} } + if v := AccessVerdict(context.Background(), lib, false, true); v.Kind != VerdictWaitKeep { + t.Fatalf("the look judges it %s, and the data on the new disk was never taken", v.Kind) + } + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder whose disk mounted over it was kept without a warrant: %+v", o) + } + if rec, _ := keptRecord(lib); rec.Kept || rec.Mount != pool { + t.Errorf("the record still says kept, or not on the new disk: %+v", rec) + } + if got := effectiveOn(t, film, radarr); got != 0 { + t.Error("an entry was added to the data on the new disk") + } +} + +// Review M3: widening a mask widens nobody else — a file made at 0644 under a default ACL carries the owning group's +// rwx under a mask of r--, and after the mesh widens the mask the owning group still reads only. +func TestWideningTheMaskGivesTheOwningGroupNothingMore(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "shows") + if err := os.Mkdir(lib, 0o775); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lib, 0o775); err != nil { + t.Fatal(err) + } + a := accessJSON("sonarr.shows", lib, "read-write", false, "mesh-sonarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + ep := filepath.Join(lib, "episode") + f, err := os.OpenFile(ep, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.Close() + groupRight := func() uint16 { + acl, _, _ := readACLs(t, ep) + g, _ := acl.perm(aclGroupObj, 0) + m, _ := acl.perm(aclMask, 0) + return g & m + } + before := groupRight() + rec, _ := keptRecord(lib) + LookKept(lib, rec.Want, SweepLimits) + if got := effectiveOn(t, ep, radarr); got&6 != 6 { + t.Fatalf("the sweep did not widen the mask: %s", permWords(got)) + } + if after := groupRight(); after != before { + t.Fatalf("the owning group may %s after the mask widened, and %s before", permWords(after), permWords(before)) + } +} + +// watchedFolder is a kept folder with a watch running on it, its users, and the end of the watch's run. +func watchedFolder(t *testing.T) (string, []store.KeptUser, *Watcher, context.CancelFunc, <-chan struct{}) { + t.Helper() + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "music") + if err := os.Mkdir(lib, 0o755); err != nil { + t.Fatal(err) + } + if _, _, err := applyAccesses(t, store.State{}, accessJSON("lidarr.music", lib, "read-write", false, + "mesh-lidarr:61001:read-write:container")); err != nil { + t.Fatal(err) + } + rec, _ := keptRecord(lib) + w, err := NewWatcher() + if err != nil { + t.Skipf("no inotify here: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + ended := make(chan struct{}) + go func() { w.Run(ctx); close(ended) }() + // The watch ends before the test's record of kept folders is put back. + t.Cleanup(func() { cancel(); <-ended }) + w.Set(map[string][]store.KeptUser{lib: rec.Want}) + return lib, rec.Want, w, cancel, ended +} + +// Review L4: a watch whose folder goes is let go (IN_IGNORED). +func TestTheWatchOfAFolderRemovedIsLetGo(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + album := filepath.Join(lib, "album") + if err := os.Mkdir(album, 0o755); err != nil { + t.Fatal(err) + } + waitFor(t, "the folder made is watched", func() bool { return w.watched() == 2 }) + if err := os.Remove(album); err != nil { + t.Fatal(err) + } + waitFor(t, "the watch of a folder removed is let go", func() bool { return w.watched() == 1 }) +} + +// Review L4: notice lost re-arms every watch, and the record says the watch is whole again. +func TestNoticeLostReArmsTheWatch(t *testing.T) { + lib, _, w, _, _ := watchedFolder(t) + w.lost() + if w.watched() != 1 { + t.Fatalf("notice lost did not re-arm the watch: %d watched", w.watched()) + } + if r, _ := keptRecord(lib); r.OutOfWatches { + t.Error("a re-armed watch is still said out of watches") + } + late := filepath.Join(lib, "late") + f, _ := os.OpenFile(late, os.O_CREATE|os.O_WRONLY, 0o644) + f.Close() + waitFor(t, "the re-armed watch repairs", func() bool { return effectiveOn(t, late, radarr)&6 == 6 }) +} + +// Review L4: users given while notices are heard are read under the lock (run with -race). +func TestUsersGivenWhileNoticesAreHeardAreReadUnderTheLock(t *testing.T) { + lib, users, w, _, _ := watchedFolder(t) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + w.Set(map[string][]store.KeptUser{lib: users}) + } + } + }() + for i := 0; i < 50; i++ { + _ = os.WriteFile(filepath.Join(lib, fmt.Sprintf("f%d", i)), nil, 0o644) + } + waitFor(t, "the last file is repaired", func() bool { + return effectiveOn(t, filepath.Join(lib, "f49"), radarr)&6 == 6 + }) + close(stop) + <-done +} + +// Review L4: the watch ends with its context, however quiet the machine. +func TestTheWatchEndsWithItsContext(t *testing.T) { + _, _, _, cancel, ended := watchedFolder(t) + cancel() + select { + case <-ended: + case <-time.After(2 * time.Second): + t.Fatal("the watch did not end with its context") + } +} + +// Review L6: a broken folder's words carry no path. +func TestABrokenFoldersWordsCarryNoPath(t *testing.T) { + dir := t.TempDir() + keeping(t, nil, nil) + file := filepath.Join(dir, "notafolder") + if err := os.WriteFile(file, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, path := range []string{file, filepath.Join(file, "below")} { + v := AccessVerdict(context.Background(), path, false, true) + if v.Kind != VerdictBroken || strings.Contains(v.Why, "/") { + t.Errorf("%s: %+v", path, v) + } + } +} + +// Review H2: a systemd .mount unit says where a disk is meant to be, as fstab does; and a mount point seen mounted +// once is remembered, so a folder below it is a missing disk on first sight when it is gone. +func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) { + units := t.TempDir() + unit := "[Unit]\nDescription=pool\n\n[Mount]\nWhat=/dev/disk/by-label/pool\nWhere=/srv/pool\nType=ext4\n" + if err := os.WriteFile(filepath.Join(units, "srv-pool.mount"), []byte(unit), 0o644); err != nil { + t.Fatal(err) + } + if got := mountUnitTargets([]string{units}); len(got) != 1 || got[0] != "/srv/pool" { + t.Fatalf("read %v from a .mount unit", got) + } + + dir := t.TempDir() + disk := filepath.Join(dir, "disk") + if err := os.Mkdir(disk, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{disk}, nil) + _ = placedOn(context.Background(), disk, disk, "", nil) // seen mounted once + MountPoints = func() []string { return []string{"/"} } + if pl := placedOn(context.Background(), filepath.Join(disk, "films"), disk, "", nil); !pl.Away { + t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk") + } +} diff --git a/internal/apply/placed.go b/internal/apply/placed.go index d96163b..7637e96 100644 --- a/internal/apply/placed.go +++ b/internal/apply/placed.go @@ -17,20 +17,49 @@ import ( // holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the // kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged // against that record: held by another mount point than the one recorded while the recorded one is not mounted is -// the disk not there. A recorded mount point that is mounted is the disk there, whatever now holds the folder (a -// mount made deeper is a placement, not a missing disk), and the record follows it. +// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is +// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made +// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's +// word as any folder holding data does (ADR 0325 §8). // // **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be: -// every mount target in /etc/fstab, every mount point a ZFS dataset names (`zfs list`, when the machine has zfs), -// and every mount point recorded for another kept folder. A target that is the folder or above it, lies below the -// mount point that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that -// holds what is there is the placed one. A mount nothing names in those places (a hand `mount` of a disk that is not -// in fstab, then unmounted before the engine ever saw the folder) is not known to be missing: said here, because the -// folder would then be made on the filesystem below it. +// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names +// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every +// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point +// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what +// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list` +// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen +// empty below it is kept as empty, and the move above takes it back the moment the disk is there. // MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test // can stand in for mounts it cannot make. var MountPoints = func() []string { + return readMountPoints() +} + +// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history"). +var mountHistory = struct { + sync.Mutex + seen map[string]bool +}{seen: map[string]bool{}} + +// sawMounted adds points to the history, and is the history: every mount point seen, never "/". +func sawMounted(points []string) []string { + mountHistory.Lock() + defer mountHistory.Unlock() + for _, p := range points { + if p != "/" { + mountHistory.seen[p] = true + } + } + out := make([]string, 0, len(mountHistory.seen)) + for p := range mountHistory.seen { + out = append(out, p) + } + return out +} + +func readMountPoints() []string { raw, err := os.ReadFile("/proc/self/mountinfo") if err != nil { return []string{"/"} @@ -60,6 +89,7 @@ func readMountTargets(ctx context.Context) []string { if raw, err := os.ReadFile("/etc/fstab"); err == nil { out = append(out, fstabTargets(string(raw))...) } + out = append(out, mountUnitTargets(systemdUnitDirs)...) if _, err := exec.LookPath("zfs"); err == nil { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -74,6 +104,33 @@ func readMountTargets(ctx context.Context) []string { return out } +// systemdUnitDirs are where systemd keeps the units a machine mounts by. +var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator", + "/usr/lib/systemd/system", "/lib/systemd/system"} + +// mountUnitTargets reads the `Where=` of every .mount unit in dirs. +func mountUnitTargets(dirs []string) []string { + var out []string + for _, dir := range dirs { + names, err := filepath.Glob(filepath.Join(dir, "*.mount")) + if err != nil { + continue + } + for _, name := range names { + raw, err := os.ReadFile(name) + if err != nil { + continue + } + for _, line := range strings.Split(string(raw), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") { + out = append(out, filepath.Clean(strings.TrimSpace(v))) + } + } + } + } + return out +} + // mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes // undone. func mountinfoPoints(raw string) []string { @@ -172,6 +229,9 @@ type placement struct { Mount string // Away is that the disk the folder was placed on is not there. Away bool + // Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what + // was kept. + Moved bool } // placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent), @@ -183,10 +243,12 @@ func placedOn(ctx context.Context, path, existing, recorded string, others []str mounted[p] = true } have := mountHolding(existing, points) + seen := sawMounted(points) if recorded != "" { - return placement{Mount: have, Away: have != recorded && !mounted[recorded]} + return placement{Mount: have, Away: have != recorded && !mounted[recorded], + Moved: have != recorded && mounted[recorded]} } - for _, t := range append(MountTargets(ctx), others...) { + for _, t := range append(append(MountTargets(ctx), others...), seen...) { if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] { return placement{Mount: have, Away: true} } diff --git a/internal/apply/walk.go b/internal/apply/walk.go index a48ece9..51d9d58 100644 --- a/internal/apply/walk.go +++ b/internal/apply/walk.go @@ -2,7 +2,6 @@ package apply import ( "errors" - "fmt" "path/filepath" "time" ) @@ -60,8 +59,8 @@ var ErrNoACL = errors.New("the filesystem allows no access entries") // never name it. func keptPath(path string) error { if !filepath.IsAbs(path) || filepath.Clean(path) != path { - return fmt.Errorf("%q is not an access's folder as the engine states it (absolute, no `..`, no doubled or "+ - "trailing separator); nothing was changed", path) + return errors.New("the folder is not stated as the engine states one (absolute, no `..`, no doubled or " + + "trailing separator); nothing was changed") } return nil } diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go index d307ff3..1f334dc 100644 --- a/internal/apply/walk_linux.go +++ b/internal/apply/walk_linux.go @@ -62,7 +62,7 @@ func openKept(path string, limits WalkLimits, stats walkStats, visit visitFunc) return nil, -1, st, errors.New("the folder is not there; nothing was changed") } if err != nil { - return nil, -1, st, fmt.Errorf("the folder is not walked: %v", err) + return nil, -1, st, errors.New("the folder is not walked: " + pathless(err)) } if mw.At != path { unix.Close(fd) @@ -363,11 +363,15 @@ func dropVisit(drop map[uint32]bool) visitFunc { continue } a, err := getACL(fd, name, mode) - if errors.Is(err, ErrNoACL) || a == nil { + if errors.Is(err, ErrNoACL) { continue } if err != nil { - return changed, false, false, err + // Counted unrepaired and said by the walk, which then walks it again: never read as "nothing to drop". + return changed, false, true, err + } + if a == nil { + continue } if left := withoutUsers(a, drop); len(left) != len(a) { if err := setACL(fd, name, left); err != nil { diff --git a/internal/apply/watch_linux.go b/internal/apply/watch_linux.go index e79ce16..8bbb43f 100644 --- a/internal/apply/watch_linux.go +++ b/internal/apply/watch_linux.go @@ -57,7 +57,8 @@ type watchedRoot struct { // NewWatcher opens the machine's notice of changes. func NewWatcher() (*Watcher, error) { - fd, err := unix.InotifyInit1(unix.IN_CLOEXEC) + // Not blocking: Run polls it, so the end of its context ends it whatever the kernel has to say. + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) if err != nil { return nil, err } @@ -149,21 +150,29 @@ func (w *Watcher) watchBelow(path string, root *watchedRoot, fd int, rel []strin } } -// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder. +// watchPoll is how long Run waits for notice before it looks whether its context ended. +const watchPoll = 250 * time.Millisecond + +// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder, and then closes the +// watch: polled, so a quiet machine's watch ends as soon as a busy one's. func (w *Watcher) Run(ctx context.Context) { - go func() { - <-ctx.Done() - unix.Close(w.fd) - }() + defer unix.Close(w.fd) buf := make([]byte, 64*1024) - for { - n, err := unix.Read(w.fd, buf) - if err != nil || n <= 0 { - if ctx.Err() != nil || !errors.Is(err, unix.EINTR) { - return - } + for ctx.Err() == nil { + ready, err := unix.Poll([]unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}}, int(watchPoll/time.Millisecond)) + if err != nil && !errors.Is(err, unix.EINTR) { + return + } + if ready <= 0 { continue } + n, err := unix.Read(w.fd, buf) + if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR) { + continue + } + if err != nil || n <= 0 { + return + } for at := 0; at+unix.SizeofInotifyEvent <= n; { ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[at])) nameAt := at + unix.SizeofInotifyEvent @@ -173,36 +182,80 @@ func (w *Watcher) Run(ctx context.Context) { w.lost() continue } + if ev.Mask&unix.IN_IGNORED != 0 { + // The watch is gone — its folder removed, or let go by Set: what pointed at it goes too. + w.forget(int(ev.Wd)) + continue + } w.heard(int(ev.Wd), name, ev.Mask) } } } -// lost is notice lost: every kept folder is said to rely on its sweep until it is watched afresh. +// lost is notice lost (the kernel's queue overflowed): every kept folder is watched afresh, and what was made +// meanwhile is the sweep's. func (w *Watcher) lost() { w.mu.Lock() defer w.mu.Unlock() for path, root := range w.roots { - root.out = true + for _, wd := range root.wds { + _, _ = unix.InotifyRmWatch(w.fd, uint32(wd)) + delete(w.dirs, wd) + } + root.wds, root.out = nil, false + w.watchTree(path, root) w.say(path, root) } } +// forget lets go of a watch the kernel removed. +func (w *Watcher) forget(wd int) { + w.mu.Lock() + defer w.mu.Unlock() + dir, ok := w.dirs[wd] + if !ok { + return + } + delete(w.dirs, wd) + if root := w.roots[dir.root]; root != nil { + for i, had := range root.wds { + if had == wd { + root.wds = append(root.wds[:i], root.wds[i+1:]...) + break + } + } + w.say(dir.root, root) + } +} + +// watched is how many folders are watched now. +func (w *Watcher) watched() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.dirs) +} + // heard repairs one entry a watch heard of: name in the watched folder wd, or the folder itself. func (w *Watcher) heard(wd int, name string, mask uint32) { w.mu.Lock() dir, ok := w.dirs[wd] root := w.roots[dir.root] + var users map[uint32]bool + var dev uint64 + if root != nil { + // Read under the lock: Set gives a root new users while notices are heard. + users, dev = root.users, root.dev + } w.mu.Unlock() if !ok || root == nil { return } - fd, err := openBelow(dir.root, dir.rel, root.dev) + fd, err := openBelow(dir.root, dir.rel, dev) if err != nil { return } defer unix.Close(fd) - visit := keepVisit(root.users) + visit := keepVisit(users) if name == "" { var st unix.Stat_t if unix.Fstat(fd, &st) == nil { @@ -211,7 +264,7 @@ func (w *Watcher) heard(wd int, name string, mask uint32) { return } var st unix.Stat_t - if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != root.dev { + if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != dev { return } isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR @@ -235,9 +288,14 @@ func (w *Watcher) heard(wd int, name string, mask uint32) { if isDir && mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 { // A folder made or moved in is watched, and what was moved in with it is the sweep's. w.mu.Lock() + defer w.mu.Unlock() + if w.roots[dir.root] != root { + // Let go by Set meanwhile: nothing more is watched for it. + unix.Close(efd) + return + } w.watchBelow(dir.root, root, efd, append(append([]string(nil), dir.rel...), name)) w.say(dir.root, root) - w.mu.Unlock() return } unix.Close(efd) From d2d9fbbd7d1cb92efaf404ebe0bd8dd91896277f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:03:02 +0200 Subject: [PATCH 03/10] Hold what mounts a missing disk, and say entries the mesh could not take out (hq ADR 0325 re-review) A container was still created over an unmounted mount point, so its writes landed on the filesystem below; a folder moved to another disk was kept on the old warrant; and entries left by a drop or a move that could not run lived only in a log line. Every container mounting an access whose disk is away now waits, the warrant holds only for the filesystem it was carried out on (the wait names the new one, path-free), entries left on a filesystem are recorded per mount and taken out when it is back, and every leftover is said as unhealthy until it is. A hung stat is shared per path, and a widened mask takes from other entries only the bits it adds. --- cmd/mesh-host/kept.go | 23 +++- cmd/mesh-host/kept_test.go | 4 +- internal/apply/acl.go | 13 +-- internal/apply/apply.go | 6 +- internal/apply/keep.go | 157 +++++++++++++++++++++++---- internal/apply/keeper_linux.go | 78 +++++++++++++- internal/apply/kept_test.go | 189 +++++++++++++++++++++++++++++++++ internal/link/messages.go | 8 ++ internal/store/kept.go | 14 +++ 9 files changed, 453 insertions(+), 39 deletions(-) diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index ba742b5..6f9ed3a 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -32,6 +32,8 @@ type judgedAccess struct { module, id, name, path string taken bool users []store.KeptUser + // leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries). + leftover bool } // accessJudge reads every access on every look. Safe for the apply and the looking loop at once. @@ -152,9 +154,10 @@ func judgeAccess(a judgedAccess) verdict { v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0) switch v.Kind { case apply.VerdictWaitKeep: - return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " + - "operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess, - Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}} + return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " + + "it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)", + waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, + What: apply.KeepWords(a.name), Placed: v.Placed}}} case apply.VerdictWaitDisk: return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)", waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module, @@ -264,13 +267,25 @@ func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) { read[i] = v } + // And the entries nobody wants that could not be taken out yet, each under the access that last named its + // folder, until its drop walk has run (ADR 0325 §8). + for _, l := range apply.Leftovers() { + as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path, + leftover: true}) + read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why}) + } + j.mu.Lock() defer j.mu.Unlock() changed := false out := make([]link.ResourceHealth, 0, len(as)) for i, a := range as { r := read[i] - v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path, + kind := link.KindAccess + if a.leftover { + kind = link.KindKeptEntries + } + v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path, State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries} if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason { v.Since = was.Since diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go index 3bf831e..c5b6817 100644 --- a/cmd/mesh-host/kept_test.go +++ b/cmd/mesh-host/kept_test.go @@ -38,7 +38,9 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { t.Fatalf("not waiting with one wait: %+v", v) } raw, _ := json.Marshal(v.waits[0]) - if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` { + want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` + + `"placed":"` + v.waits[0].Placed + `"}` + if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") { t.Fatalf("the wait reads %s", raw) } if strings.Contains(v.reason, dir) { diff --git a/internal/apply/acl.go b/internal/apply/acl.go index 6fbab09..2b8e8c5 100644 --- a/internal/apply/acl.go +++ b/internal/apply/acl.go @@ -142,7 +142,7 @@ func (a acl) check() error { // keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask // widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps -// what it gave every other entry, each capped at the old mask when it widens; with none yet (a file the mode alone described) the mask starts at the owning +// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning // group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave. // Nothing else is changed. func keptACL(a acl, need map[uint32]uint16) acl { @@ -159,16 +159,17 @@ func keptACL(a acl, need map[uint32]uint16) acl { if i := out.find(aclMask, 0); i >= 0 { old := out[i].Perm out[i].Perm |= widen - if out[i].Perm != old { + if added := out[i].Perm &^ old; added != 0 { // **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does // not set here were limited by the old mask: a file made at 0644 under a default ACL carries the - // owning group's rwx with a mask of r--. Each is set to what it gave under the old mask, so its - // effective right stays exactly as it was. + // owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask + // now adds cannot keep its effective right without losing that bit: exactly those bits are taken off + // it, and nothing else of it changes. An entry holding none of them is not touched at all. for j := range out { e := &out[j] _, ours := need[e.ID] - if e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours) { - e.Perm &= old + if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 { + e.Perm &^= added } } } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 31e4aaa..4440a27 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1135,11 +1135,11 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru return begin(r), err } // Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9). - if id := mountsMissing(res, in.accesses); id != "" { + if id := mountsMissing(ctx, res, in.accesses); id != "" { out := begin(r) out.Action = "waiting" - out.Detail = "not created: it mounts the access " + id + ", whose folder is not there; created once it " + - "is (novox/hq ADR 0325 §9)" + out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " + + "created once it is (novox/hq ADR 0325 §9)" return out, nil } } diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 440f5eb..1a6be6d 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -2,6 +2,8 @@ package apply import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "os" @@ -249,10 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } + // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a + // folder moved to another is not taken, and asks again (§8). + takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved)) switch { case rec.Kept && !pl.Moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} - case taken: + case takenHere: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved} case facts.Empty: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved} @@ -262,6 +267,45 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved} } +// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left +// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the +// controller's word is not read for the new one, and nothing else of what was kept there holds. +func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { + left := f.Left + if len(f.Users) > 0 && f.Mount != "" { + left = addLeft(left, f.Mount, f.Users) + } + return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, + Left: left} +} + +// addLeft adds users' entries left on mount, once per mount and uid. +func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft { + for i := range left { + if left[i].Mount == mount { + have := usersOf(left[i].Users) + for _, u := range users { + if _, had := have[u.UID]; !had { + left[i].Users = append(left[i].Users, u) + } + } + return left + } + } + return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)}) +} + +// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the +// folder's filesystem is not the one a warrant was given for. +func PlacedID(mount string) string { + if mount == "" { + return "" + } + source, fstype := mountSourceOf(mount) + sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype)) + return "fs-" + hex.EncodeToString(sum[:6]) +} + // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose // server is down blocks a stat for ever, and one access must never hold the apply. var statBound = 5 * time.Second @@ -270,21 +314,41 @@ var statBound = 5 * time.Second var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)") // lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own. -func lstatWithin(path string) (os.FileInfo, error) { return within(os.Lstat, path) } -func statWithin(path string) (os.FileInfo, error) { return within(os.Stat, path) } -func within(stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { - type read struct { - info os.FileInfo - err error +func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) } +func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) } + +// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead +// mount holds one goroutine per path, never one per apply or look. +type statRead struct { + done chan struct{} + info os.FileInfo + err error +} + +var statsInFlight = struct { + sync.Mutex + reads map[string]*statRead +}{reads: map[string]*statRead{}} + +func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + key := op + " " + path + statsInFlight.Lock() + r, running := statsInFlight.reads[key] + if !running { + r = &statRead{done: make(chan struct{})} + statsInFlight.reads[key] = r + go func() { + r.info, r.err = stat(path) + statsInFlight.Lock() + delete(statsInFlight.reads, key) + statsInFlight.Unlock() + close(r.done) + }() } - done := make(chan read, 1) - go func() { - info, err := stat(path) - done <- read{info, err} - }() + statsInFlight.Unlock() select { - case r := <-done: + case <-r.done: return r.info, r.err case <-time.After(statBound): return nil, errNoAnswer @@ -364,7 +428,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O // What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty // below it): judged afresh, the record of what was kept there no longer holds. _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - *f = store.KeptFolder{Path: path, Mount: v.mount, Want: f.Want, Act: f.Act} + *f = movedRecord(*f, v.mount) return had }) } @@ -405,7 +469,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O return out, nil } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Want = v.mount, users + f.Mount, f.Want, f.IDs = v.mount, users, want.ids return true }) out.Action = "waiting" @@ -447,7 +511,11 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O var pending string _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { first := !f.Kept - f.Kept, f.Mount, f.Want = true, v.mount, users + f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + if v.how == store.KeptByWarrant && f.TakenOn == "" { + // The controller says it was taken, and this engine has no record of where: here, now. + f.TakenOn = v.mount + } if first { f.How, f.At = v.how, time.Now().UTC() // A folder that held nothing has nothing under it to walk; one the controller says was taken @@ -537,8 +605,8 @@ func forgetUndeclared(folders map[string]*folderWant) { } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { f.Want = nil - // A folder never kept, or kept with nobody's entries left, is forgotten now. - return f.Kept && len(f.Users) > 0 + // A folder never kept, or kept with nobody's entries left anywhere, is forgotten now. + return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0 }) } } @@ -590,7 +658,7 @@ func accessPaths(resources []declaration.Resource, refused map[string]error) map // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026). -func mountsMissing(c *declaration.Container, accesses map[string]string) string { +func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string { for _, v := range c.Volumes { src := mountSource(v) if !strings.HasPrefix(src, "/") { @@ -603,6 +671,12 @@ func mountsMissing(c *declaration.Container, accesses map[string]string) string if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) { return id } + // There, and its disk not: what is at the path is the empty mount point on the filesystem below, and + // a container's writes would land there (§9). + rec, _ := keptRecord(path) + if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away { + return id + } } } return "" @@ -620,7 +694,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool { return out } for _, r := range d.Resources { - if c, ok := r.(*declaration.Container); ok && mountsMissing(c, accesses) != "" { + if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" { out[c.ID] = true } } @@ -639,9 +713,10 @@ const ( VerdictNobody = "nobody" ) -// AccessJudgement is a look's judgement of an access's folder, and why, in words. +// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at +// it for a wait, with no path (PlacedID). type AccessJudgement struct { - Kind, Why string + Kind, Why, Placed string } // AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the @@ -662,7 +737,43 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud case v.kind == folderMake: return AccessJudgement{Kind: VerdictMake} case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken: - return AccessJudgement{Kind: VerdictWaitKeep} + return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)} } return AccessJudgement{Kind: VerdictKeep} } + +// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder, +// why, in words with no path. +type Leftover struct { + Module, Access, Name, Path, Why string +} + +// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names +// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each +// is said until its drop walk has run. +func Leftovers() []Leftover { + all, err := KeptFolders.All() + if err != nil { + return nil + } + var out []Leftover + for path, f := range all { + if len(f.IDs) == 0 { + continue + } + module, name := accessName(f.IDs[0]) + dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting" + switch { + case dropping: + out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " + + "0325 §8)"}) + case len(f.Left) > 0: + out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + name + " on a filesystem no longer at its folder could not be removed: they are removed once that " + + "filesystem is back (novox/hq ADR 0325 §8)"}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access }) + return out +} diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 5faeb71..f19c20e 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -61,6 +61,12 @@ func ReconcileKept(ctx context.Context) []string { if ctx.Err() != nil { break } + if len(f.Left) > 0 { + said = append(said, dropLeft(path, f)...) + if f, _ = keptRecord(path); f.Path == "" { + continue + } + } if !f.Kept || sameUsers(f.Users, f.Want) { continue } @@ -69,6 +75,69 @@ func ReconcileKept(ctx context.Context) []string { return said } +// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds +// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they +// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and +// said (Leftovers). +func dropLeft(path string, f store.KeptFolder) []string { + lock := folderLock(path) + lock.Lock() + defer lock.Unlock() + holding := "" + if _, err := lstatWithin(path); err == nil { + holding = mountHolding(path, MountPoints()) + } + var said []string + var still []store.KeptLeft + var mine []store.KeptUser + for _, l := range f.Left { + switch { + case f.Kept && l.Mount == f.Mount: + mine = append(mine, l.Users...) + case l.Mount != holding: + still = append(still, l) + default: + drop := map[uint32]bool{} + for _, u := range l.Users { + drop[u.UID] = true + } + w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop)) + if err != nil || w.Failed != "" { + still = append(still, l) + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed))) + continue + } + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+ + "were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files)) + } + } + _ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool { + k.Left = still + for _, u := range mine { + if _, has := usersOf(k.Users)[u.UID]; !has { + k.Users = append(k.Users, u) + } + } + return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0) + }) + return said +} + +func pathlessOr(err error) string { + if err == nil { + return "" + } + return pathless(err) +} + +func firstOf(a, b string) string { + if a != "" { + return a + } + return b +} + func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string { lock := folderLock(path) lock.Lock() @@ -196,10 +265,15 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } started := time.Now().UTC() _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { - if !f.Kept || v.moved { + if v.moved { + *f = movedRecord(*f, v.mount) + } + if !f.Kept { f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil } - f.Warrant, f.By, f.Mount, f.Want = warrant, by, v.mount, users + f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids + // The warrant is for the filesystem it was carried out on (§8). + f.TakenOn = v.mount f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} return true }) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 57a7efa..1dc6b20 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -861,3 +862,191 @@ func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) { t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk") } } + +// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset. +func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) { + t.Helper() + dir := aclFolder(t) + dataset = filepath.Join(dir, "tank") + if err := os.Mkdir(dataset, 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{dataset}, nil) + lib = filepath.Join(dataset, "movies") + if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false, + users...)); err != nil { + t.Fatal(err) + } + return dataset, lib +} + +// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount +// point is at the path: its writes would land on the filesystem below (§9). +func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) { + _, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + MountPoints = func() []string { return []string{"/"} } + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + if len(args) > 0 && args[0] == "container" { + return "", errors.New("no such container") + } + return "", nil + } + d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+ + `,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`) + report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) + if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" { + t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o) + } + for _, r := range ran { + if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") { + t.Fatalf("the container was started: %s", r) + } + } + if !WaitingOnAccess(d)["radarr.app"] { + t.Error("liveness is not told the container waits on its access") + } +} + +// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until +// the drop walk can run; then it is forgotten. +func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { + dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container") + other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}` + if _, _, err := applyAccesses(t, store.State{}, other); err != nil { + t.Fatal(err) + } + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + left := Leftovers() + if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") || + strings.Contains(left[0].Why, dataset) { + t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left) + } + MountPoints = func() []string { return []string{"/", dataset} } + ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("said after the drop ran: %+v", left) + } + if _, had := keptRecord(lib); had { + t.Error("a folder nobody uses was not forgotten once its entries were out") + } + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { + t.Error("the entries were not taken out") + } +} + +// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that +// filesystem holds the folder again, they are taken out. +func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(lib, 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") { + t.Fatalf("entries left on the filesystem below are not said: %+v", left) + } + // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { + t.Fatal("the entries left on the filesystem that came back were not taken out") + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("still said after they were taken out: %+v", left) + } +} + +// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on; +// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked. +func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + before := AccessVerdict(context.Background(), lib, true, true) + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o) + } + // And on the next apply too, its record now on the new filesystem. + report, _, _ = applyAccesses(t, store.State{}, taken) + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("the second apply kept it on the old warrant: %+v", o) + } + v := AccessVerdict(context.Background(), lib, true, true) + if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") { + t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before) + } + ReconcileKept(context.Background()) + if got := effectiveOn(t, film, radarr); got != 0 { + t.Fatal("the data on the new filesystem was walked") + } +} + +// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile. +func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) { + was := statBound + statBound = 20 * time.Millisecond + defer func() { statBound = was }() + release := make(chan struct{}) + defer close(release) + var started int32 + var mu sync.Mutex + hang := func(string) (os.FileInfo, error) { + mu.Lock() + started++ + mu.Unlock() + <-release + return nil, nil + } + for i := 0; i < 5; i++ { + if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) { + t.Fatalf("a hung read answered: %v", err) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one path were started while the first hung", started) + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 88083ca..ab77a04 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -309,6 +309,11 @@ const KindDirectory = "directory" // it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12). const KindAccess = "access" +// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet +// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at +// a folder. Unhealthy, said under the access that last named it, until the drop walk has run. +const KindKeptEntries = "kept-entries" + // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" @@ -445,6 +450,9 @@ type AccessWait struct { Access string `json:"access"` Part string `json:"part"` What string `json:"what"` + // Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant + // given for another filesystem (the folder moved) does not answer it, and the controller asks again. + Placed string `json:"placed,omitempty"` } // KeptEntry is one user of a kept folder as its listing says it. diff --git a/internal/store/kept.go b/internal/store/kept.go index ed63fbc..8965692 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -59,6 +59,14 @@ type KeptFolder struct { // declared. Where they differ a walk adds or removes exactly the difference. Users []KeptUser `json:"users,omitempty"` Want []KeptUser `json:"want,omitempty"` + // IDs are the accesses that last named the folder: what a leftover is said under once none does. + IDs []string `json:"ids,omitempty"` + // TakenOn is the mount point the operator's warrant was carried out on (ADR 0325 §8): the controller's word + // that the folder was taken holds for that filesystem alone, and a folder moved to another asks again. + TakenOn string `json:"taken_on,omitempty"` + // Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or + // one gone): taken out by a drop walk once that filesystem holds the folder again, said until then. + Left []KeptLeft `json:"left,omitempty"` // Act is the last walk under the folder: a take, an add or a drop. Act *KeptAct `json:"act,omitempty"` // Sweep is how far the sweep got (ADR 0325 §10). @@ -69,6 +77,12 @@ type KeptFolder struct { OutOfWatches bool `json:"out_of_watches,omitempty"` } +// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are. +type KeptLeft struct { + Mount string `json:"mount"` + Users []KeptUser `json:"users"` +} + // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. type KeptAct struct { // Act is take, add or drop. From f0adb177f03fa183b8c67ab0e719f3ca1b44e773 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:06:34 +0200 Subject: [PATCH 04/10] Leave no leftover for a folder kept only while empty, and name filesystems by more than their mount point (hq ADR 0325 third review) A folder kept because it was empty, then covered by its disk, was said as unhealthy leftover entries for ever: its entries sit on a hidden empty folder and grant nothing, so they are now forgotten, and only entries walked onto data under a warrant are kept to be taken out. A warrant and a kept folder now name their filesystem by mount point, source and type, so another pool at the same point is a move; records naming a mount point are read as its name. A record with no access ids is said under a path-free name. --- internal/apply/keep.go | 76 +++++++++++++------ internal/apply/keeper_linux.go | 4 +- internal/apply/kept_test.go | 131 ++++++++++++++++++++++++++++----- internal/apply/placed.go | 6 +- internal/store/kept.go | 8 +- 5 files changed, 181 insertions(+), 44 deletions(-) diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 1a6be6d..7b4231a 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -244,6 +244,9 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if pl.Away { return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} } + // Another filesystem at the same mount point (another pool, another disk) is a move too. + here := PlacedID(pl.Mount) + moved := pl.Moved || (rec.Placed != "" && rec.Mount == pl.Mount && rec.Placed != here) facts, err := readFolder(path) if err != nil { return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} @@ -253,30 +256,41 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a // folder moved to another is not taken, and asks again (§8). - takenHere := taken && (rec.TakenOn == pl.Mount || (rec.TakenOn == "" && !pl.Moved)) + takenHere := taken && (takenOnID(rec.TakenOn) == here || (rec.TakenOn == "" && !moved)) switch { - case rec.Kept && !pl.Moved: + case rec.Kept && !moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} case takenHere: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: pl.Moved} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: moved} case facts.Empty: - return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: pl.Moved} + return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: moved} } // What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty // below it included (a move): it waits for the operator's word. - return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: pl.Moved} + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved} } -// movedRecord is f judged afresh on the filesystem now at mount: what the mesh added on the one it kept is left -// to be taken out once that filesystem holds the folder again (Left), the warrant's filesystem is kept so the -// controller's word is not read for the new one, and nothing else of what was kept there holds. +// movedRecord is f judged afresh on the filesystem now at mount: the warrant's filesystem is kept so the +// controller's word is not read for the new one, and nothing else of what was kept there holds. What the mesh +// added on the old filesystem is left to be taken out once it holds the folder again (Left) only where it was +// walked onto data under a warrant: a folder kept because it was empty, or made, carries entries on that empty +// folder alone, hidden under the mount point and granting nothing on data, and they are forgotten (said once, by +// the apply that saw the move). func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { left := f.Left - if len(f.Users) > 0 && f.Mount != "" { + if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" { left = addLeft(left, f.Mount, f.Users) } - return store.KeptFolder{Path: f.Path, Mount: mount, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, - Left: left} + return store.KeptFolder{Path: f.Path, Mount: mount, Placed: PlacedID(mount), Want: f.Want, IDs: f.IDs, + Act: f.Act, TakenOn: f.TakenOn, Left: left} +} + +// takenOnID is a record's TakenOn as a filesystem's name: one written before names held the mount point itself. +func takenOnID(takenOn string) string { + if strings.HasPrefix(takenOn, "/") { + return PlacedID(takenOn) + } + return takenOn } // addLeft adds users' entries left on mount, once per mount and uid. @@ -435,6 +449,10 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O note := "" if v.moved { note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh" + if rec.How != store.KeptByWarrant { + note += "; the entries on the empty folder kept below are forgotten: hidden under the mount, they grant " + + "nothing on data" + } } if len(r.Grant) > 0 { note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" @@ -469,7 +487,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O return out, nil } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Want, f.IDs = v.mount, users, want.ids + f.Mount, f.Placed, f.Want, f.IDs = v.mount, PlacedID(v.mount), users, want.ids return true }) out.Action = "waiting" @@ -497,7 +515,7 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O changed, err := keepTop(path, usersOf(users)) if errors.Is(err, ErrNoACL) { _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Want = v.mount, users + f.Mount, f.Placed, f.Want = v.mount, PlacedID(v.mount), users return true }) out.Action = "waiting" @@ -511,11 +529,12 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O var pending string _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { first := !f.Kept - f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + f.Kept, f.Mount, f.Placed, f.Want, f.IDs = true, v.mount, PlacedID(v.mount), users, want.ids if v.how == store.KeptByWarrant && f.TakenOn == "" { // The controller says it was taken, and this engine has no record of where: here, now. - f.TakenOn = v.mount + f.TakenOn = PlacedID(v.mount) } + f.TakenOn = takenOnID(f.TakenOn) if first { f.How, f.At = v.how, time.Now().UTC() // A folder that held nothing has nothing under it to walk; one the controller says was taken @@ -758,18 +777,15 @@ func Leftovers() []Leftover { } var out []Leftover for path, f := range all { - if len(f.IDs) == 0 { - continue - } - module, name := accessName(f.IDs[0]) + module, name, access := recordName(path, f) dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting" switch { case dropping: - out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " + "0325 §8)"}) case len(f.Left) > 0: - out = append(out, Leftover{Module: module, Access: f.IDs[0], Name: name, Path: path, Why: "the mesh's entries for " + + out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + name + " on a filesystem no longer at its folder could not be removed: they are removed once that " + "filesystem is back (novox/hq ADR 0325 §8)"}) } @@ -777,3 +793,21 @@ func Leftovers() []Leftover { sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access }) return out } + +// recordName is the module, the name and the access a kept folder's record is said under: the accesses that last +// named it, or for a record written before it kept them, a name with no path — the module of a container identity +// among its users, and a folder named by a digest of its path. +func recordName(path string, f store.KeptFolder) (module, name, access string) { + if len(f.IDs) > 0 { + module, name = accessName(f.IDs[0]) + return module, name, f.IDs[0] + } + sum := sha256.Sum256([]byte(path)) + name = "folder-" + hex.EncodeToString(sum[:4]) + for _, u := range append(append([]store.KeptUser(nil), f.Users...), f.Want...) { + if m, ok := strings.CutPrefix(u.Identity, declaration.IdentityPrefix); ok && u.From == declaration.FromContainer { + return m, name, m + "." + name + } + } + return "", name, name +} diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index f19c20e..67da4ae 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -272,8 +272,8 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil } f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids - // The warrant is for the filesystem it was carried out on (§8). - f.TakenOn = v.mount + // The warrant is for the filesystem it was carried out on (§8), named as a wait names it. + f.Placed, f.TakenOn = PlacedID(v.mount), PlacedID(v.mount) f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} return true }) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 1dc6b20..4982c4c 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -937,9 +937,51 @@ func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { } } -// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that -// filesystem holds the folder again, they are taken out. +// Review L3: a disk mounted over a folder taken below it on a warrant leaves the mesh's entries on that data +// recorded and said; once that filesystem holds the folder again, they are taken out. func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, a), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + film := filepath.Join(lib, "film") + if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + _ = unix.Removexattr(film, aclAccessXattr) + MountPoints = func() []string { return []string{"/", pool} } + if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + t.Fatal(err) + } + if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") { + t.Fatalf("entries left on the data taken below are not said: %+v", left) + } + // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. + MountPoints = func() []string { return []string{"/"} } + ReconcileKept(context.Background()) + if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, filepath.Join(lib, "old")); return a.perm(aclUser, radarr) }(); still { + t.Fatal("the entries left on the filesystem that came back were not taken out") + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("still said after they were taken out: %+v", left) + } +} + +// Review (third) M: a folder kept only because it was empty leaves nothing said when a disk mounts over it — its +// entries sit on an empty folder hidden under the mount and grant nothing on data. +func TestAFolderKeptEmptyLeavesNoLeftoverWhenADiskMountsOverIt(t *testing.T) { dir := aclFolder(t) keeping(t, nil, nil) pool := filepath.Join(dir, "pool") @@ -951,26 +993,79 @@ func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *te if _, _, err := applyAccesses(t, store.State{}, a); err != nil { t.Fatal(err) } - film := filepath.Join(lib, "film") - if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { - t.Fatal(err) - } - _ = unix.Removexattr(film, aclAccessXattr) MountPoints = func() []string { return []string{"/", pool} } - if _, _, err := applyAccesses(t, store.State{}, a); err != nil { + report, _, err := applyAccesses(t, store.State{}, a) + if err != nil { t.Fatal(err) } - if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") { - t.Fatalf("entries left on the filesystem below are not said: %+v", left) - } - // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. - MountPoints = func() []string { return []string{"/"} } - ReconcileKept(context.Background()) - if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still { - t.Fatal("the entries left on the filesystem that came back were not taken out") - } if left := Leftovers(); len(left) != 0 { - t.Fatalf("still said after they were taken out: %+v", left) + t.Fatalf("a folder kept while empty is said as leftover entries for ever: %+v", left) + } + if rec, _ := keptRecord(lib); len(rec.Left) != 0 { + t.Errorf("entries on the hidden empty folder are recorded: %+v", rec.Left) + } + if o := outcomeOf(report, "radarr.media"); !strings.Contains(o.Detail, "forgotten") { + t.Errorf("the forgetting is not said: %+v", o) + } +} + +// Review (third) low: a record written before it kept the accesses' ids is said too, under a name with no path. +func TestALeftoverWithNoAccessIdsIsSaidUnderAPathFreeName(t *testing.T) { + keeping(t, nil, nil) + path := "/srv/media/old" + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.Kept, f.Users = true, []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}} + f.Act = &store.KeptAct{State: "waiting"} + return true + }) + left := Leftovers() + if len(left) != 1 || left[0].Module != "radarr" || strings.Contains(left[0].Why, path) || + strings.Contains(left[0].Access, "/") { + t.Fatalf("a record without ids is not said, path-free: %+v", left) + } +} + +// Review (third) low: another filesystem mounted at the same point is not the one the warrant was for. +func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { + dir := aclFolder(t) + pool := filepath.Join(dir, "pool") + lib := filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{pool}, nil) + source := "tank/a" + was := MountSource + MountSource = func(point string) (string, string) { + if point == pool { + return source, "zfs" + } + return "", "" + } + defer func() { MountSource = was }() + notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") + if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done + taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { + t.Fatal("the folder taken waits on its own filesystem") + } + source = "tank/b" // another pool imported at the same mount point + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("another filesystem at the same mount point was kept on the old warrant: %+v", o) + } + if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { + t.Fatalf("the look judges it %s", v.Kind) } } diff --git a/internal/apply/placed.go b/internal/apply/placed.go index 7637e96..8008bda 100644 --- a/internal/apply/placed.go +++ b/internal/apply/placed.go @@ -147,7 +147,11 @@ func mountinfoPoints(raw string) []string { // mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its // name. Empty when the machine says nothing of it. -func mountSourceOf(point string) (source, fstype string) { +func mountSourceOf(point string) (source, fstype string) { return MountSource(point) } + +// MountSource is mountSourceOf as the machine answers it. A variable so a test can stand in for another filesystem +// mounted at the same point. +var MountSource = func(point string) (source, fstype string) { raw, err := os.ReadFile("/proc/self/mountinfo") if err != nil { return "", "" diff --git a/internal/store/kept.go b/internal/store/kept.go index 8965692..34049cc 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -61,8 +61,12 @@ type KeptFolder struct { Want []KeptUser `json:"want,omitempty"` // IDs are the accesses that last named the folder: what a leftover is said under once none does. IDs []string `json:"ids,omitempty"` - // TakenOn is the mount point the operator's warrant was carried out on (ADR 0325 §8): the controller's word - // that the folder was taken holds for that filesystem alone, and a folder moved to another asks again. + // Placed names the filesystem the folder was kept on (the engine's PlacedID: its mount point, source and type), + // so another filesystem mounted at the same point is a move, as another mount point is. + Placed string `json:"placed,omitempty"` + // TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the + // controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another + // asks again. A record written before held the mount point here, and is read as that mount's name. TakenOn string `json:"taken_on,omitempty"` // Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or // one gone): taken out by a drop walk once that filesystem holds the folder again, said until then. From 21cd0362320d7963081ddd969be8f43d765cdce9 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:06:57 +0200 Subject: [PATCH 05/10] Format the third review's tests (hq ADR 0325) --- internal/apply/kept_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 4982c4c..3f13c75 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -971,7 +971,10 @@ func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *te // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. MountPoints = func() []string { return []string{"/"} } ReconcileKept(context.Background()) - if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, filepath.Join(lib, "old")); return a.perm(aclUser, radarr) }(); still { + if _, still := func() (uint16, bool) { + a, _, _ := readACLs(t, filepath.Join(lib, "old")) + return a.perm(aclUser, radarr) + }(); still { t.Fatal("the entries left on the filesystem that came back were not taken out") } if left := Leftovers(); len(left) != 0 { From f00f9c2a2f99cd3b87cfcf00cfc4f05fd8e74ccb Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:10:18 +0200 Subject: [PATCH 06/10] Name a filesystem by its kernel id, never its device name (hq ADR 0325 fourth review) An ext4 or xfs library mounted by UUID can come back from a reboot as another device (/dev/sdb1 as /dev/sdc1); naming the filesystem by mountinfo's source read that as a move, stripped the warranted library's entries and asked for a new Approve. A filesystem is now named by statfs's f_fsid (from the UUID, or the ZFS dataset), its type and its mount point. Entries left on the filesystem a folder is on now are merged back or held, never walked away, and a TakenOn naming a mount point is not taken. --- internal/apply/keep.go | 42 ++++++++------ internal/apply/keeper_linux.go | 14 ++++- internal/apply/kept_test.go | 101 +++++++++++++++++++++++++++++---- internal/apply/walk_linux.go | 10 ++++ internal/apply/walk_other.go | 3 + internal/store/kept.go | 9 ++- 6 files changed, 147 insertions(+), 32 deletions(-) diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 7b4231a..5f50759 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -256,7 +256,7 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken } // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a // folder moved to another is not taken, and asks again (§8). - takenHere := taken && (takenOnID(rec.TakenOn) == here || (rec.TakenOn == "" && !moved)) + takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved)) switch { case rec.Kept && !moved: return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How} @@ -279,24 +279,16 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { left := f.Left if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" { - left = addLeft(left, f.Mount, f.Users) + left = addLeft(left, f.Mount, f.Placed, f.Users) } return store.KeptFolder{Path: f.Path, Mount: mount, Placed: PlacedID(mount), Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, Left: left} } -// takenOnID is a record's TakenOn as a filesystem's name: one written before names held the mount point itself. -func takenOnID(takenOn string) string { - if strings.HasPrefix(takenOn, "/") { - return PlacedID(takenOn) - } - return takenOn -} - // addLeft adds users' entries left on mount, once per mount and uid. -func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []store.KeptLeft { +func addLeft(left []store.KeptLeft, mount, placed string, users []store.KeptUser) []store.KeptLeft { for i := range left { - if left[i].Mount == mount { + if left[i].Mount == mount && left[i].Placed == placed { have := usersOf(left[i].Users) for _, u := range users { if _, had := have[u.UID]; !had { @@ -306,20 +298,37 @@ func addLeft(left []store.KeptLeft, mount string, users []store.KeptUser) []stor return left } } - return append(left, store.KeptLeft{Mount: mount, Users: append([]store.KeptUser(nil), users...)}) + return append(left, store.KeptLeft{Mount: mount, Placed: placed, Users: append([]store.KeptUser(nil), users...)}) } // PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the -// folder's filesystem is not the one a warrant was given for. +// folder's filesystem is not the one a warrant was given for. It is built from what stays the same across reboots +// for one filesystem — the kernel's filesystem id (statfs f_fsid: ext4 and xfs derive it from the filesystem's +// UUID, ZFS from the dataset), the filesystem's type and its mount point — and never from the device name +// mountinfo gives as its source, which a reboot may renumber (/dev/sdb1 becoming /dev/sdc1). func PlacedID(mount string) string { if mount == "" { return "" } - source, fstype := mountSourceOf(mount) - sum := sha256.Sum256([]byte(mount + "\x00" + source + "\x00" + fstype)) + _, fstype := mountSourceOf(mount) + fsid := fsidWithin(mount) + sum := sha256.Sum256([]byte(mount + "\x00" + fsid + "\x00" + fstype)) return "fs-" + hex.EncodeToString(sum[:6]) } +// fsidWithin is MountFsid within statBound: empty when it does not answer, so a mount that is down is named by its +// point and type alone, never held on. +func fsidWithin(mount string) string { + done := make(chan string, 1) + go func() { done <- MountFsid(mount) }() + select { + case id := <-done: + return id + case <-time.After(statBound): + return "" + } +} + // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose // server is down blocks a stat for ever, and one access must never hold the apply. var statBound = 5 * time.Second @@ -534,7 +543,6 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O // The controller says it was taken, and this engine has no record of where: here, now. f.TakenOn = PlacedID(v.mount) } - f.TakenOn = takenOnID(f.TakenOn) if first { f.How, f.At = v.how, time.Now().UTC() // A folder that held nothing has nothing under it to walk; one the controller says was taken diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 67da4ae..5c6e028 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -87,14 +87,24 @@ func dropLeft(path string, f store.KeptFolder) []string { if _, err := lstatWithin(path); err == nil { holding = mountHolding(path, MountPoints()) } + holdingID := "" + if holding != "" { + holdingID = PlacedID(holding) + } var said []string var still []store.KeptLeft var mine []store.KeptUser for _, l := range f.Left { + same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed) switch { - case f.Kept && l.Mount == f.Mount: + case f.Kept && same: + // Left on the filesystem the folder is kept on now: the folder's own entries. mine = append(mine, l.Users...) - case l.Mount != holding: + case same: + // On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's + // once it is kept there again. + still = append(still, l) + case l.Mount != holding || (l.Placed != "" && l.Placed != holdingID): still = append(still, l) default: drop := map[uint32]bool{} diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 3f13c75..53239fa 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -1028,24 +1028,31 @@ func TestALeftoverWithNoAccessIdsIsSaidUnderAPathFreeName(t *testing.T) { } } -// Review (third) low: another filesystem mounted at the same point is not the one the warrant was for. -func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { +// placedPool is a folder of a library on a pool the test says is mounted, with the pool's device name and filesystem +// id as the test sets them, taken under a warrant. +func placedPool(t *testing.T, source, fsid *string) (pool, lib, taken string) { + t.Helper() dir := aclFolder(t) - pool := filepath.Join(dir, "pool") - lib := filepath.Join(pool, "media") + pool = filepath.Join(dir, "pool") + lib = filepath.Join(pool, "media") if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { t.Fatal(err) } keeping(t, []string{pool}, nil) - source := "tank/a" - was := MountSource + wasSource, wasFsid := MountSource, MountFsid MountSource = func(point string) (string, string) { if point == pool { - return source, "zfs" + return *source, "ext4" } return "", "" } - defer func() { MountSource = was }() + MountFsid = func(point string) string { + if point == pool { + return *fsid + } + return "" + } + t.Cleanup(func() { MountSource, MountFsid = wasSource, wasFsid }) notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { t.Fatal(err) @@ -1055,11 +1062,18 @@ func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { t.Fatal(err) } <-done - taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") + taken = accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container") if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { t.Fatal("the folder taken waits on its own filesystem") } - source = "tank/b" // another pool imported at the same mount point + return pool, lib, taken +} + +// Review (third) low: another filesystem mounted at the same point is not the one the warrant was for. +func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + _, lib, taken := placedPool(t, &source, &fsid) + fsid = "1111222233334444" // another filesystem mounted at the same point report, _, err := applyAccesses(t, store.State{}, taken) if err != nil { t.Fatal(err) @@ -1072,6 +1086,28 @@ func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { } } +// Review (fourth): the same filesystem whose device the kernel renamed across a reboot (/dev/sdb1 to /dev/sdc1, one +// filesystem id) is the same filesystem: still taken, still kept, and none of its entries walked away. +func TestADeviceRenamedIsTheSameFilesystem(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + _, lib, taken := placedPool(t, &source, &fsid) + source = "/dev/sdc1" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("a device renamed reads as another filesystem: %+v", o) + } + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("the library's entries were walked away: the identity may %s", permWords(got)) + } + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a device renamed leaves entries said as left: %+v", left) + } +} + // Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on; // moved to another, it is not taken, the wait names the new filesystem, and nothing is walked. func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { @@ -1148,3 +1184,48 @@ func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) { t.Fatalf("%d reads of one path were started while the first hung", started) } } + +// Review (fourth): entries left on the filesystem the folder is on now are never walked away — merged back into +// the folder's users when it is kept there, held while it is not. +func TestEntriesLeftOnTheFilesystemTheFolderIsOnAreNeverWalkedAway(t *testing.T) { + source, fsid := "/dev/sdb1", "0a0b0c0d01020304" + pool, lib, _ := placedPool(t, &source, &fsid) + rec, _ := keptRecord(lib) + users := rec.Users + for _, kept := range []bool{false, true} { + _ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool { + f.Kept, f.Users = kept, nil + f.Left = []store.KeptLeft{{Mount: pool, Placed: f.Placed, Users: users}} + return true + }) + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("kept=%v: entries on the folder's own filesystem were walked away", kept) + } + r, _ := keptRecord(lib) + if kept && (len(r.Left) != 0 || len(r.Users) != 1) { + t.Errorf("kept: not merged back into the folder's users: %+v", r) + } + if !kept && len(r.Left) != 1 { + t.Errorf("not kept: the entries' record was dropped: %+v", r) + } + } +} + +// Review (fourth): a TakenOn naming a mount point (an unmerged build's record) is not taken on whatever is mounted +// there now. +func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) { + dir := aclFolder(t) + keeping(t, nil, nil) + lib := filepath.Join(dir, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + _ = KeptFolders.Update(lib, func(f *store.KeptFolder, _ bool) bool { + f.Mount, f.TakenOn = "/", "/" + return true + }) + if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { + t.Fatalf("an old-form record is read as taken: %s", v.Kind) + } +} diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go index 1f334dc..411162c 100644 --- a/internal/apply/walk_linux.go +++ b/internal/apply/walk_linux.go @@ -538,3 +538,13 @@ func createPath(path string, stats walkStats) (int, error) { unix.Close(fd) return made, nil } + +// MountFsid is the kernel's id of the filesystem at point (statfs f_fsid), in hex; empty when it cannot be read. +// A variable so a test can stand in for a filesystem it cannot mount. +var MountFsid = func(point string) string { + var st unix.Statfs_t + if err := unix.Statfs(point, &st); err != nil { + return "" + } + return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1])) +} diff --git a/internal/apply/walk_other.go b/internal/apply/walk_other.go index c341303..1a29965 100644 --- a/internal/apply/walk_other.go +++ b/internal/apply/walk_other.go @@ -53,3 +53,6 @@ type Watcher struct{} func NewWatcher() (*Watcher, error) { return nil, errNoKeeping } func (*Watcher) Set(map[string][]store.KeptUser) {} func (*Watcher) Run(context.Context) {} + +// MountFsid reads no filesystem id where a folder is not kept. +var MountFsid = func(string) string { return "" } diff --git a/internal/store/kept.go b/internal/store/kept.go index 34049cc..3dfb6a4 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -66,7 +66,7 @@ type KeptFolder struct { Placed string `json:"placed,omitempty"` // TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the // controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another - // asks again. A record written before held the mount point here, and is read as that mount's name. + // asks again. One that names no filesystem this way (a mount point, from an unmerged build) is not taken. TakenOn string `json:"taken_on,omitempty"` // Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or // one gone): taken out by a drop walk once that filesystem holds the folder again, said until then. @@ -83,8 +83,11 @@ type KeptFolder struct { // KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are. type KeptLeft struct { - Mount string `json:"mount"` - Users []KeptUser `json:"users"` + Mount string `json:"mount"` + // Placed names that filesystem as KeptFolder.Placed does: entries left on the filesystem the folder is on + // now are the folder's own, never walked away. + Placed string `json:"placed,omitempty"` + Users []KeptUser `json:"users"` } // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. From 46c16e18febc809adbb38515263a58b96194d757 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:14:42 +0200 Subject: [PATCH 07/10] Name a filesystem by its UUID, its kernel id or its source, and keep nothing that has no name (hq ADR 0325 fifth review) NFS and FUSE report a zero f_fsid, so another export or another FUSE mount at the same point was named alike and kept without a warrant; xfs's f_fsid follows the device number. A filesystem is now named by its UUID (its device matched in /dev/disk/by-uuid), else a nonzero f_fsid, else mountinfo's source; one that cannot be named is never kept or taken. Entries left on a filesystem not back within 30 days are forgotten, said once, and the naming read is shared per mount. --- internal/apply/keep.go | 154 ++++++++++++++++++++++----------- internal/apply/keeper_linux.go | 25 +++++- internal/apply/kept_test.go | 153 ++++++++++++++++++++++++++++++++ internal/apply/walk_linux.go | 22 +++++ internal/apply/walk_other.go | 3 + internal/store/kept.go | 8 ++ 6 files changed, 313 insertions(+), 52 deletions(-) diff --git a/internal/apply/keep.go b/internal/apply/keep.go index 5f50759..dbd31d9 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -203,6 +203,8 @@ type folderVerdict struct { err error // moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh. moved bool + // unnamed is a folder whose filesystem cannot be named: never kept, nor taken. + unnamed bool } // assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then @@ -244,9 +246,10 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if pl.Away { return folderVerdict{kind: folderWaitDisk, mount: pl.Mount} } - // Another filesystem at the same mount point (another pool, another disk) is a move too. + // Another filesystem at the same mount point (another pool, another disk) is a move too. A record from an earlier + // build names its filesystem another way, and is not compared: the next keep names it this way. here := PlacedID(pl.Mount) - moved := pl.Moved || (rec.Placed != "" && rec.Mount == pl.Mount && rec.Placed != here) + moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here) facts, err := readFolder(path) if err != nil { return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))} @@ -254,6 +257,11 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } + // A filesystem that cannot be named is never kept: what is there could be anyone's (§8). + if here == "" { + return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved, unnamed: true, + why: "its filesystem cannot be named (no UUID, no filesystem id, no source)"} + } // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a // folder moved to another is not taken, and asks again (§8). takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved)) @@ -279,14 +287,21 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken func movedRecord(f store.KeptFolder, mount string) store.KeptFolder { left := f.Left if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" { - left = addLeft(left, f.Mount, f.Placed, f.Users) + left = addLeft(left, f.Mount, f.Placed, f.Fs, f.Users) } - return store.KeptFolder{Path: f.Path, Mount: mount, Placed: PlacedID(mount), Want: f.Want, IDs: f.IDs, + placed, fs := placeOf(mount) + return store.KeptFolder{Path: f.Path, Mount: mount, Placed: placed, Fs: fs, Want: f.Want, IDs: f.IDs, Act: f.Act, TakenOn: f.TakenOn, Left: left} } +// placeOf is the name of the filesystem at mount, and what it was made from. +func placeOf(mount string) (placed, fs string) { + fs = filesystemKey(mount) + return placedFrom(mount, fs), fs +} + // addLeft adds users' entries left on mount, once per mount and uid. -func addLeft(left []store.KeptLeft, mount, placed string, users []store.KeptUser) []store.KeptLeft { +func addLeft(left []store.KeptLeft, mount, placed, fs string, users []store.KeptUser) []store.KeptLeft { for i := range left { if left[i].Mount == mount && left[i].Placed == placed { have := usersOf(left[i].Users) @@ -298,35 +313,57 @@ func addLeft(left []store.KeptLeft, mount, placed string, users []store.KeptUser return left } } - return append(left, store.KeptLeft{Mount: mount, Placed: placed, Users: append([]store.KeptUser(nil), users...)}) + return append(left, store.KeptLeft{Mount: mount, Placed: placed, Fs: fs, Since: time.Now().UTC(), + Users: append([]store.KeptUser(nil), users...)}) } // PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the -// folder's filesystem is not the one a warrant was given for. It is built from what stays the same across reboots -// for one filesystem — the kernel's filesystem id (statfs f_fsid: ext4 and xfs derive it from the filesystem's -// UUID, ZFS from the dataset), the filesystem's type and its mount point — and never from the device name -// mountinfo gives as its source, which a reboot may renumber (/dev/sdb1 becoming /dev/sdc1). -func PlacedID(mount string) string { - if mount == "" { +// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder +// on it is then never kept (it fails closed, and waits). +func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) } + +// placedFrom is PlacedID from a filesystem key already read. +func placedFrom(mount, key string) string { + if mount == "" || key == "" { return "" } - _, fstype := mountSourceOf(mount) - fsid := fsidWithin(mount) - sum := sha256.Sum256([]byte(mount + "\x00" + fsid + "\x00" + fstype)) + sum := sha256.Sum256([]byte(mount + "\x00" + key)) return "fs-" + hex.EncodeToString(sum[:6]) } -// fsidWithin is MountFsid within statBound: empty when it does not answer, so a mount that is down is named by its -// point and type alone, never held on. -func fsidWithin(mount string) string { - done := make(chan string, 1) - go func() { done <- MountFsid(mount) }() - select { - case id := <-done: - return id - case <-time.After(statBound): +// filesystemKey is what names the filesystem at mount across reboots, in this order: +// +// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in +// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first; +// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its +// UUID; +// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own +// source string (both report f_fsid 0). +// +// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of +// these answers within statBound: then nothing is kept there. Read once per mount at a time. +func filesystemKey(mount string) string { + if mount == "" { return "" } + v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) }) + if !ok { + return "" + } + return v.(string) +} + +func readFilesystemKey(mount string) string { + if u := MountUUID(mount); u != "" { + return "uuid:" + u + } + if id := MountFsid(mount); strings.Trim(id, "0") != "" { + return "fsid:" + id + } + if source, fstype := mountSourceOf(mount); source != "" && source != "none" { + return "source:" + fstype + ":" + source + } + return "" } // statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose @@ -341,41 +378,57 @@ var errNoAnswer = errors.New("it does not answer: a read of it did not return wi func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) } func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) } -// statRead is one read of a path in flight, shared by everyone who asks while it is: a read that hangs on a dead -// mount holds one goroutine per path, never one per apply or look. -type statRead struct { +// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds +// one goroutine per path, never one per apply or look. +type flight struct { done chan struct{} - info os.FileInfo - err error + v any } -var statsInFlight = struct { +var inFlight = struct { sync.Mutex - reads map[string]*statRead -}{reads: map[string]*statRead{}} + reads map[string]*flight +}{reads: map[string]*flight{}} -func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { - key := op + " " + path - statsInFlight.Lock() - r, running := statsInFlight.reads[key] +// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not +// answer in time. +func boundedOnce(key string, read func() any) (any, bool) { + inFlight.Lock() + f, running := inFlight.reads[key] if !running { - r = &statRead{done: make(chan struct{})} - statsInFlight.reads[key] = r + f = &flight{done: make(chan struct{})} + inFlight.reads[key] = f go func() { - r.info, r.err = stat(path) - statsInFlight.Lock() - delete(statsInFlight.reads, key) - statsInFlight.Unlock() - close(r.done) + f.v = read() + inFlight.Lock() + delete(inFlight.reads, key) + inFlight.Unlock() + close(f.done) }() } - statsInFlight.Unlock() + inFlight.Unlock() select { - case <-r.done: - return r.info, r.err + case <-f.done: + return f.v, true case <-time.After(statBound): + return nil, false + } +} + +func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) { + type read struct { + info os.FileInfo + err error + } + v, ok := boundedOnce(op+" "+path, func() any { + info, err := stat(path) + return read{info, err} + }) + if !ok { return nil, errNoAnswer } + r := v.(read) + return r.info, r.err } // absPath is an absolute path in words: what pathless takes out. @@ -496,7 +549,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O return out, nil } _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Placed, f.Want, f.IDs = v.mount, PlacedID(v.mount), users, want.ids + f.Mount, f.Want, f.IDs = v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) return true }) out.Action = "waiting" @@ -524,7 +578,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O changed, err := keepTop(path, usersOf(users)) if errors.Is(err, ErrNoACL) { _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { - f.Mount, f.Placed, f.Want = v.mount, PlacedID(v.mount), users + f.Mount, f.Want = v.mount, users + f.Placed, f.Fs = placeOf(v.mount) return true }) out.Action = "waiting" @@ -538,7 +593,8 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O var pending string _ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool { first := !f.Kept - f.Kept, f.Mount, f.Placed, f.Want, f.IDs = true, v.mount, PlacedID(v.mount), users, want.ids + f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids + f.Placed, f.Fs = placeOf(v.mount) if v.how == store.KeptByWarrant && f.TakenOn == "" { // The controller says it was taken, and this engine has no record of where: here, now. f.TakenOn = PlacedID(v.mount) diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 5c6e028..8e710e2 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -75,6 +75,9 @@ func ReconcileKept(ctx context.Context) []string { return said } +// LeftFor is how long entries left on a filesystem are kept recorded, and said, while it is not back. +var LeftFor = 30 * 24 * time.Hour + // dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds // it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they // are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and @@ -95,7 +98,20 @@ func dropLeft(path string, f store.KeptFolder) []string { var still []store.KeptLeft var mine []store.KeptUser for _, l := range f.Left { - same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed) + // On the folder's own filesystem now: the same name, or a name of an earlier build's form at the same + // mount point (which cannot be compared, and is never walked away). + same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed || l.Fs == "") + back := l.Mount == holding && (l.Placed == "" || l.Placed == holdingID) + if l.Since.IsZero() { + l.Since = time.Now().UTC() + } + if !same && !back && time.Since(l.Since) > LeftFor { + // **A filesystem not back within LeftFor is gone** (a disk replaced, a pool destroyed): what was left + // on it is forgotten, said once, rather than said unhealthy for ever. + said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem not back for %s "+ + "are forgotten: it is taken for gone", path, LeftFor)) + continue + } switch { case f.Kept && same: // Left on the filesystem the folder is kept on now: the folder's own entries. @@ -104,7 +120,7 @@ func dropLeft(path string, f store.KeptFolder) []string { // On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's // once it is kept there again. still = append(still, l) - case l.Mount != holding || (l.Placed != "" && l.Placed != holdingID): + case !back: still = append(still, l) default: drop := map[uint32]bool{} @@ -253,6 +269,8 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra return nil, "", errors.New(RootWords(v.why)) case v.kind == folderMake: return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody") + case v.unnamed: + return nil, "", errors.New(v.why + ", so a warrant could name no filesystem it was given for") } lock := folderLock(path) if !lock.TryLock() { @@ -283,7 +301,8 @@ func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warra } f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids // The warrant is for the filesystem it was carried out on (§8), named as a wait names it. - f.Placed, f.TakenOn = PlacedID(v.mount), PlacedID(v.mount) + f.Placed, f.Fs = placeOf(v.mount) + f.TakenOn = f.Placed f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started} return true }) diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 53239fa..3fab036 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -1229,3 +1229,156 @@ func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) { t.Fatalf("an old-form record is read as taken: %s", v.Kind) } } + +// poolNamed is a library on a pool the test says is mounted, whose UUID, filesystem id and mountinfo source are as +// the test sets them; the access declared, not taken. +func poolNamed(t *testing.T, uuid, fsid, source *string) (pool, lib, access string) { + t.Helper() + dir := aclFolder(t) + pool = filepath.Join(dir, "pool") + lib = filepath.Join(pool, "media") + if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil { + t.Fatal(err) + } + keeping(t, []string{pool}, nil) + wasUUID, wasFsid, wasSource := MountUUID, MountFsid, MountSource + MountUUID = func(p string) string { return pick(p == pool, *uuid) } + MountFsid = func(p string) string { return pick(p == pool, *fsid) } + MountSource = func(p string) (string, string) { return pick(p == pool, *source), pick(p == pool, "nfs4") } + t.Cleanup(func() { MountUUID, MountFsid, MountSource = wasUUID, wasFsid, wasSource }) + return pool, lib, accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container") +} + +func pick(yes bool, v string) string { + if yes { + return v + } + return "x" +} + +// takeIt takes the folder of access under a warrant, and says it is kept. +func takeIt(t *testing.T, lib, access string) { + t.Helper() + if _, _, err := applyAccesses(t, store.State{}, access); err != nil { + t.Fatal(err) + } + done, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w-1", "op", noServices) + if err != nil { + t.Fatal(err) + } + <-done +} + +// Review (fifth) M: an NFS export (f_fsid 0, no UUID) is named by its source; another export at the same mount +// point is another filesystem, not taken. +func TestAnNFSExportIsNamedByItsSource(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "nas:/export/media" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + if report, _, _ := applyAccesses(t, store.State{}, taken); outcomeOf(report, "radarr.media").Action == "waiting" { + t.Fatal("the export taken waits on its own filesystem") + } + source = "other-nas:/export/media" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("another export at the same mount point was kept on the old warrant: %+v", o) + } +} + +// Review (fifth) M: a filesystem that cannot be named is never kept, nor taken: it fails closed, and waits. +func TestAFilesystemThatCannotBeNamedIsNeverKept(t *testing.T) { + uuid, fsid, source := "", "0000000000000000", "" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + if err := os.RemoveAll(filepath.Join(lib, "old")); err != nil { // empty: it would be kept without asking + t.Fatal(err) + } + report, _, err := applyAccesses(t, store.State{}, access) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { + t.Fatalf("a folder on a filesystem that cannot be named was kept: %+v", o) + } + if got := effectiveOn(t, lib, radarr); got != 0 { + t.Fatal("an entry was added on a filesystem that cannot be named") + } + if _, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w", "op", noServices); err == nil { + t.Fatal("the take act ran on a filesystem that cannot be named") + } +} + +// Review (fifth): xfs's f_fsid follows the device number; its UUID names it first, so a renumbered device is the same +// filesystem. +func TestAFilesystemsUUIDNamesItBeforeItsFsid(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + fsid, source = "0000082100000000", "/dev/sdc1" + report, _, err := applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("a renumbered device with one UUID reads as another filesystem: %+v", o) + } + ReconcileKept(context.Background()) + if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { + t.Fatalf("the library's entries were walked away: %s", permWords(got)) + } +} + +// Review (fifth): entries left on a filesystem that never comes back are forgotten after LeftFor, said once, and no +// longer said unhealthy. +func TestEntriesLeftOnAFilesystemThatNeverReturnsAreForgotten(t *testing.T) { + keeping(t, nil, nil) + path := "/srv/media/gone" + _ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool { + f.IDs = []string{"radarr.media"} + f.Left = []store.KeptLeft{{Mount: "/srv/media", Placed: "fs-000000000000", Fs: "uuid:gone", + Since: time.Now().Add(-LeftFor - time.Hour), + Users: []store.KeptUser{{Identity: "mesh-radarr", UID: radarr, Use: "read-write", From: "container"}}}} + return true + }) + if len(Leftovers()) != 1 { + t.Fatal("the leftover is not said before it expires") + } + said := ReconcileKept(context.Background()) + if left := Leftovers(); len(left) != 0 { + t.Fatalf("a leftover on a filesystem gone for longer than %s is still said: %+v", LeftFor, left) + } + if !strings.Contains(strings.Join(said, "\n"), "forgotten") { + t.Errorf("the forgetting is not said: %v", said) + } +} + +// Review (fifth) low: naming a filesystem whose read hangs holds one read per mount, whoever asks meanwhile. +func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) { + was, wasBound := MountUUID, statBound + statBound = 20 * time.Millisecond + release := make(chan struct{}) + var mu sync.Mutex + started := 0 + MountUUID = func(string) string { + mu.Lock() + started++ + mu.Unlock() + <-release + return "" + } + defer func() { close(release); MountUUID, statBound = was, wasBound }() + for i := 0; i < 5; i++ { + if id := PlacedID("/a/dead/mount"); id != "" { + t.Fatalf("a filesystem that does not answer is named %s", id) + } + } + mu.Lock() + defer mu.Unlock() + if started != 1 { + t.Fatalf("%d reads of one mount were started while the first hung", started) + } +} diff --git a/internal/apply/walk_linux.go b/internal/apply/walk_linux.go index 411162c..692b312 100644 --- a/internal/apply/walk_linux.go +++ b/internal/apply/walk_linux.go @@ -548,3 +548,25 @@ var MountFsid = func(point string) string { } return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1])) } + +// MountUUID is the UUID of the filesystem at point when it is on a block device: the name in /dev/disk/by-uuid of +// the device the mount's own device number is. Empty for one on no block device (NFS, FUSE, ZFS, a btrfs +// subvolume's own device). A variable so a test can stand in for disks it cannot make. +var MountUUID = func(point string) string { + var st unix.Stat_t + if err := unix.Stat(point, &st); err != nil { + return "" + } + const dir = "/dev/disk/by-uuid" + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + for _, e := range entries { + var dev unix.Stat_t + if unix.Stat(dir+"/"+e.Name(), &dev) == nil && dev.Mode&unix.S_IFMT == unix.S_IFBLK && dev.Rdev == st.Dev { + return e.Name() + } + } + return "" +} diff --git a/internal/apply/walk_other.go b/internal/apply/walk_other.go index 1a29965..7d9c79a 100644 --- a/internal/apply/walk_other.go +++ b/internal/apply/walk_other.go @@ -56,3 +56,6 @@ func (*Watcher) Run(context.Context) {} // MountFsid reads no filesystem id where a folder is not kept. var MountFsid = func(string) string { return "" } + +// MountUUID reads no UUID where a folder is not kept. +var MountUUID = func(string) string { return "" } diff --git a/internal/store/kept.go b/internal/store/kept.go index 3dfb6a4..4e9252e 100644 --- a/internal/store/kept.go +++ b/internal/store/kept.go @@ -64,6 +64,10 @@ type KeptFolder struct { // Placed names the filesystem the folder was kept on (the engine's PlacedID: its mount point, source and type), // so another filesystem mounted at the same point is a move, as another mount point is. Placed string `json:"placed,omitempty"` + // Fs is what Placed was made from beside the mount point: "uuid:", "fsid:" + // or "source::". Kept on this machine only, never said. Absent on a record from an + // earlier build, whose Placed is then not compared. + Fs string `json:"fs,omitempty"` // TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the // controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another // asks again. One that names no filesystem this way (a mount point, from an unmerged build) is not taken. @@ -87,7 +91,11 @@ type KeptLeft struct { // Placed names that filesystem as KeptFolder.Placed does: entries left on the filesystem the folder is on // now are the folder's own, never walked away. Placed string `json:"placed,omitempty"` + Fs string `json:"fs,omitempty"` Users []KeptUser `json:"users"` + // Since is when they were left: a filesystem not back at its mount point within LeftFor is gone, and what was + // left on it is forgotten (said once). + Since time.Time `json:"since,omitzero"` } // KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path. From 2ff40109ac3d7cceb8f47358bf70403f24c6d5a3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:20:53 +0200 Subject: [PATCH 08/10] Read a filesystem that could not be named as unknown, never as a move (hq ADR 0325 sixth review) A name read that timed out gave an empty name, which differed from the recorded one and so reset the record: a folder kept while empty asked again, a warranted one was walked again. An unnamed filesystem now leaves the folder exactly as it is, the access is said not known for that look, and it resumes once it is named. --- cmd/mesh-host/kept.go | 3 ++ internal/apply/keep.go | 22 +++++++++--- internal/apply/keeper_linux.go | 2 ++ internal/apply/kept_test.go | 66 +++++++++++++++++++++++++++++++++- 4 files changed, 87 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index 6f9ed3a..b50151c 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -164,6 +164,9 @@ func judgeAccess(a judgedAccess) verdict { What: apply.DiskWords(a.name)}}} case apply.VerdictRoot: return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)} + case apply.VerdictUnknown: + // Not a fault and not the operator's: said as not known, and judged again on the next look. + return verdict{state: link.StateUnknown, reason: v.Why + "; judged again on the next look"} case apply.VerdictBroken: // Its words carry no path (AccessVerdict says them so): the reason reaches the operator's channel. return verdict{state: link.StateUnhealthy, reason: v.Why} diff --git a/internal/apply/keep.go b/internal/apply/keep.go index dbd31d9..dad5a6f 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -189,6 +189,8 @@ const ( folderWaitDisk = "wait-disk" // folderRoot is root's data on its placed filesystem: never kept. folderRoot = "root" + // folderUnknown is a folder whose filesystem could not be named: left as it is until it is. + folderUnknown = "unknown" ) // folderVerdict is the judgement of one folder, with what it read. @@ -249,6 +251,13 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken // Another filesystem at the same mount point (another pool, another disk) is a move too. A record from an earlier // build names its filesystem another way, and is not compared: the next keep names it this way. here := PlacedID(pl.Mount) + // **A filesystem that cannot be named now is unknown, never a move** (a name read that did not answer, a + // filesystem with no UUID, id or source): the folder holds what it is — nothing walked, nothing newly kept, + // nothing reset — and is judged again when it is named. One never kept is so never kept: it fails closed. + if here == "" { + return folderVerdict{kind: folderUnknown, mount: pl.Mount, unnamed: true, + why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"} + } moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here) facts, err := readFolder(path) if err != nil { @@ -257,11 +266,6 @@ func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken if why := rootOwned(facts.UID, facts.GID); why != "" { return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why} } - // A filesystem that cannot be named is never kept: what is there could be anyone's (§8). - if here == "" { - return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved, unnamed: true, - why: "its filesystem cannot be named (no UUID, no filesystem id, no source)"} - } // The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a // folder moved to another is not taken, and asks again (§8). takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved)) @@ -520,6 +524,10 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)" } switch v.kind { + case folderUnknown: + out.Action = "waiting" + out.Detail = v.why + ": the folder is left exactly as it is, and judged again on the next apply" + note + return out, nil case folderWaitDisk: out.Action = "waiting" out.Detail = "the disk that holds it is not there: nothing is made, and nothing mounts it until it is " + @@ -792,6 +800,8 @@ const ( VerdictWaitDisk = "wait-disk" VerdictRoot = "root" VerdictBroken = "broken" + // VerdictUnknown is a folder whose filesystem could not be named this look: neither kept nor faulted. + VerdictUnknown = "unknown" // VerdictNobody is a folder no identity uses: nothing is kept, and only a missing disk is said. VerdictNobody = "nobody" ) @@ -813,6 +823,8 @@ func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJud return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)} case v.kind == folderWaitDisk: return AccessJudgement{Kind: VerdictWaitDisk} + case v.kind == folderUnknown: + return AccessJudgement{Kind: VerdictUnknown, Why: v.why} case !used: return AccessJudgement{Kind: VerdictNobody} case v.kind == folderRoot: diff --git a/internal/apply/keeper_linux.go b/internal/apply/keeper_linux.go index 8e710e2..a4f6afc 100644 --- a/internal/apply/keeper_linux.go +++ b/internal/apply/keeper_linux.go @@ -179,6 +179,8 @@ func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string why = pathless(v.err) case v.kind == folderRoot: why = RootWords(v.why) + case v.kind == folderUnknown: + why = v.why case v.moved || v.kind == folderWaitKeep: why = "what is at it now is not what the mesh kept there" } diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 3fab036..326be71 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "time" @@ -1370,7 +1371,7 @@ func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) { <-release return "" } - defer func() { close(release); MountUUID, statBound = was, wasBound }() + defer func() { close(release); flightsDone(t); MountUUID, statBound = was, wasBound }() for i := 0; i < 5; i++ { if id := PlacedID("/a/dead/mount"); id != "" { t.Fatalf("a filesystem that does not answer is named %s", id) @@ -1382,3 +1383,66 @@ func TestNamingAHungFilesystemIsOneReadPerMount(t *testing.T) { t.Fatalf("%d reads of one mount were started while the first hung", started) } } + +// Review (sixth): a name read that does not answer is unknown, never a move — a folder kept while empty and filled +// since is not asked about again, a warranted one is not walked again, and nothing of either record is reset. +func TestAFilesystemNotNamedThisTimeIsNeverAMove(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + // The name's reads hang past their bound while hanging is set; installed once, before anything reads it. + var hanging atomic.Bool + release := make(chan struct{}) + named := MountUUID + MountUUID = func(p string) string { + if hanging.Load() { + <-release + } + return named(p) + } + takeIt(t, lib, access) + taken := strings.Replace(access, `"taken":false`, `"taken":true`, 1) + before, _ := keptRecord(lib) + + wasBound := statBound + statBound = 20 * time.Millisecond + hanging.Store(true) + report, _, err := applyAccesses(t, store.State{}, taken) + hanging.Store(false) + close(release) + flightsDone(t) + statBound = wasBound + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" || !strings.Contains(o.Detail, "could not be named") { + t.Fatalf("a folder whose filesystem was not named this time is not said so: %+v", o) + } + after, _ := keptRecord(lib) + if !after.Kept || after.Placed != before.Placed || after.TakenOn != before.TakenOn || len(after.Left) != 0 || + !sameUsers(after.Users, before.Users) { + t.Fatalf("the record was reset by a name not read:\nbefore %+v\nafter %+v", before, after) + } + + // Named again: kept as it was, with nothing asked and nothing walked. + waitFor(t, "the hung read returned", func() bool { return PlacedID(filepath.Dir(lib)) == before.Placed }) + report, _, err = applyAccesses(t, store.State{}, taken) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { + t.Fatalf("named again, the folder asks again: %+v", o) + } + if r, _ := keptRecord(lib); r.Act == nil || r.Act.Act != "take" { + t.Errorf("the folder was walked again: %+v", r.Act) + } +} + +// flightsDone waits until no read of a hung test is in flight: it goes on reading the stand-ins after its release. +func flightsDone(t *testing.T) { + t.Helper() + waitFor(t, "the hung reads returned", func() bool { + inFlight.Lock() + defer inFlight.Unlock() + return len(inFlight.reads) == 0 + }) +} From c85c2804a010a005903331a18c3933aacf7b81eb Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:24:24 +0200 Subject: [PATCH 09/10] Apply an older controller's accesses exactly as before (hq ADR 0325) The engine delivers before the controller half, and today's controller sends no identities, users or taken. Keeping now starts only for an access whose folder has users, or a declaration with identities; every other access is mounted when present and refused when absent as on main, with nothing made, kept, held or judged. A leftover whose walk could not run says why, not always that its disk is away. --- cmd/mesh-host/kept.go | 4 + cmd/mesh-host/kept_test.go | 21 +++++ internal/apply/apply.go | 15 ++-- internal/apply/keep.go | 76 ++++++++++++++++-- internal/apply/oldshape_test.go | 134 ++++++++++++++++++++++++++++++++ 5 files changed, 239 insertions(+), 11 deletions(-) create mode 100644 internal/apply/oldshape_test.go diff --git a/cmd/mesh-host/kept.go b/cmd/mesh-host/kept.go index b50151c..d435a77 100644 --- a/cmd/mesh-host/kept.go +++ b/cmd/mesh-host/kept.go @@ -83,6 +83,10 @@ func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) { if !ok || held[a.ID] || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) { continue } + // An older controller's access is judged by nothing, as before ADR 0325: no state is said of it. + if !apply.Keeps(d, a.Path) { + continue + } at := strings.LastIndex(a.ID, ".") if at <= 0 { continue diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go index c5b6817..6c9c5dc 100644 --- a/cmd/mesh-host/kept_test.go +++ b/cmd/mesh-host/kept_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/link" @@ -47,3 +48,23 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { t.Fatalf("the reason names the path: %s", v.reason) } } + +// An older controller's accesses (no users, no identities) are judged by nothing, as before ADR 0325: the statement +// carries no resource for them, waiting, unknown or kept-entries. +func TestAnOlderControllersAccessesSayNothing(t *testing.T) { + dir := t.TempDir() + d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"radarr.library","type":"access","path":"` + filepath.Join(dir, "library") + `","mode":"read-write"}, + {"id":"plex.movies","type":"access","path":"` + filepath.Join(dir, "absent") + `"}]}`)) + if err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(dir, "library"), 0o755); err != nil { + t.Fatal(err) + } + j := newAccessJudge() + j.set(d, nil) + if vs, _ := j.look(time.Now()); len(vs) != 0 { + t.Fatalf("an older controller's accesses are judged: %+v", vs) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 4440a27..0934dcd 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -351,8 +351,10 @@ func ApplyMindingWindows( case declaration.TypeAccess: // The folder is left as it is: what the mesh added to it is taken out by the walk once no access names // it (novox/hq ADR 0325 §8), never by removing the access's record. - action, detail = "forgotten", "an operator-owned folder is never the host's to remove; the entries "+ - "the mesh added go once no access names it" + action, detail = "forgotten", "an operator-owned path is never the host's to remove" + if _, kept := keptRecord(filepath.Clean(orphan.Target)); kept { + detail += "; the entries the mesh added go once no access names it" + } gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed) if said != "" { detail = joinDetail(detail, said) @@ -540,7 +542,8 @@ func ApplyMindingWindows( // What every declared access asks of its folder, several accesses of one folder as one (novox/hq ADR 0325 §5), // and every accessed folder, which no container mounts while it is not there (§9). in.folders = foldersOf(d.Resources, in.refused) - in.accesses = accessPaths(d.Resources, in.refused) + in.identities = len(d.Identities) > 0 + in.accesses = accessPaths(d.Resources, in.refused, in.identities) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} for _, resource := range d.Resources { @@ -2193,8 +2196,10 @@ type inputs struct { dirsBefore map[string]bool // folders is what every declared access asks of its folder, by path (novox/hq ADR 0325 §5); accesses every // accessed folder with its access's id, which no container mounts while it is not there (§9). - folders map[string]*folderWant - accesses map[string]string + folders map[string]*folderWant + // identities is that the declaration carries identities: its controller speaks ADR 0325 for every access. + identities bool + accesses map[string]string // handed is every group a record that went this apply handed to a resource still asking for it. handed Handed } diff --git a/internal/apply/keep.go b/internal/apply/keep.go index dad5a6f..27d5afc 100644 --- a/internal/apply/keep.go +++ b/internal/apply/keep.go @@ -497,6 +497,9 @@ func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (O out := begin(r) path := filepath.Clean(r.Path) want := in.folders[path] + if !keeps(want, in.identities) { + return legacyAccess(r) + } users := want.list() _, name := accessName(r.ID) rec, _ := keptRecord(path) @@ -737,16 +740,39 @@ func NearestAncestor(path string) (MissingPath, error) { } // accessPaths is every declared access's folder with its id: what no container mounts while it is not there. -func accessPaths(resources []declaration.Resource, refused map[string]error) map[string]string { +// +// Only accesses the mesh keeps (Keeps): an older controller's access holds no container, as before ADR 0325. +func accessPaths(resources []declaration.Resource, refused map[string]error, identities bool) map[string]string { + folders := foldersOf(resources, refused) out := map[string]string{} for _, r := range resources { - if a, ok := r.(*declaration.Access); ok && refused[filepath.Clean(a.Path)] == nil { - out[filepath.Clean(a.Path)] = a.ID + a, ok := r.(*declaration.Access) + if !ok { + continue + } + if path := filepath.Clean(a.Path); refused[path] == nil && keeps(folders[path], identities) { + out[path] = a.ID } } return out } +// keeps is whether ADR 0325's keeping applies to a folder: some access naming it carries users, or the +// declaration carries identities — a controller that speaks link.IdentityContract. An older controller's access +// (no users, no identities) is applied exactly as before: mounted when present, refused when absent, nothing kept, +// nothing made, nothing judged (ADR 0051 §7). +func keeps(f *folderWant, identities bool) bool { + return identities || (f != nil && len(f.users) > 0) +} + +// Keeps is keeps for a whole declaration's access at path, for the engine's looks. +func Keeps(d *declaration.Declaration, path string) bool { + if d == nil { + return false + } + return keeps(foldersOf(d.Resources, nil)[filepath.Clean(path)], len(d.Identities) > 0) +} + // mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there // now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026). func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string { @@ -780,7 +806,7 @@ func WaitingOnAccess(d *declaration.Declaration) map[string]bool { if d == nil { return out } - accesses := accessPaths(d.Resources, refusedPlaces(d.Resources)) + accesses := accessPaths(d.Resources, refusedPlaces(d.Resources), len(d.Identities) > 0) if len(accesses) == 0 { return out } @@ -857,9 +883,13 @@ func Leftovers() []Leftover { dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting" switch { case dropping: + // Why the walk could not run, as it recorded it: its disk not there, or its filesystem not named. + why := strings.TrimSuffix(strings.TrimPrefix(f.Act.Why, "not walked: "), "; walked once it is") + if why == "" { + why = "the walk could not run" + } out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + - name + " could not be removed: its disk is not there; they are removed once it is (novox/hq ADR " + - "0325 §8)"}) + name + " could not be removed: " + why + "; they are removed once it can (novox/hq ADR 0325 §8)"}) case len(f.Left) > 0: out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " + name + " on a filesystem no longer at its folder could not be removed: they are removed once that " + @@ -887,3 +917,37 @@ func recordName(path string, f store.KeptFolder) (module, name, access string) { } return "", name, name } + +// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7). +// +// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container +// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026 +// records. So the host checks first and says plainly that the operator must provide the path, rather than +// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports +// a change: the machine did not move, the host merely confirmed a fact about it. +func legacyAccess(r *declaration.Access) (Outcome, error) { + out := begin(r) + info, err := os.Stat(r.Path) + if errors.Is(err, os.ErrNotExist) { + return out, fmt.Errorf( + "%s is not there, and the mesh does not own it — the operator must provide it. It is "+ + "shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+ + "nothing, so a missing one is said here rather than made as root by the container "+ + "runtime", r.Path) + } + if err != nil { + return out, err + } + if !info.IsDir() { + return out, fmt.Errorf( + "%s is not a directory, and an access is a shared directory the operator provides", + r.Path) + } + mode := declaration.AccessRead + if r.Mode != "" { + mode = r.Mode + } + out.Action = "unchanged" + out.Detail = "operator-owned; present, " + mode + ", nothing managed" + return out, nil +} diff --git a/internal/apply/oldshape_test.go b/internal/apply/oldshape_test.go new file mode 100644 index 0000000..a9d23da --- /dev/null +++ b/internal/apply/oldshape_test.go @@ -0,0 +1,134 @@ +//go:build linux + +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// The engine delivers before the controller half (novox/hq ADR 0325): a controller that does not speak +// link.IdentityContract sends no identities, no users and no taken, and every access of its declarations is applied +// exactly as before this engine — mounted when present, refused when absent (ADR 0051 §7), nothing kept, nothing +// made, no container held, nothing judged. These are origin/main's outcomes, word for word. +func TestAnOlderControllersAccessesAreAppliedExactlyAsBefore(t *testing.T) { + keeping(t, nil, nil) + dir := t.TempDir() + unwritable := filepath.Join(dir, "library") // its group may not write it + empty := filepath.Join(dir, "spool") + absent := filepath.Join(dir, "media", "movies") + for _, d := range []string{unwritable, empty} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(d, 0o755); err != nil { + t.Fatal(err) + } + } + // A root-owned empty folder, read only. + rootEmpty := "" + for _, c := range []string{"/var/empty", "/srv/http", "/srv/ftp"} { + if entries, err := os.ReadDir(c); err == nil && len(entries) == 0 { + if uid, _, _ := statOwner(mustLstat(t, c)); uid == 0 { + rootEmpty = c + break + } + } + } + resources := []string{ + `{"id":"radarr.library","type":"access","path":"` + unwritable + `","mode":"read-write"}`, + `{"id":"sab.spool","type":"access","path":"` + empty + `","mode":"read-write"}`, + `{"id":"plex.movies","type":"access","path":"` + absent + `"}`, + `{"id":"plex.app","type":"container","name":"plex","image":"` + pinned + `","volumes":["` + absent + `:/movies"]}`, + } + if rootEmpty != "" { + resources = append(resources, `{"id":"web.root","type":"access","path":"`+rootEmpty+`","mode":"read-write"}`) + } + var started bool + run := func(_ context.Context, name string, args ...string) (string, error) { + if len(args) > 0 && args[0] == "run" { + started = true + } + if len(args) > 0 && args[0] == "container" { + if started { + return "true\t" + containerSpec(declare(t, resources[3]).Resources[0].(*declaration.Container), inputs{}), nil + } + return "", errors.New("no such container") + } + return "", nil + } + report, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")), store.State{}, + store.OriginDeclared, run, nil, nil) + + if err == nil || !strings.Contains(err.Error(), absent+" is not there, and the mesh does not own it — the operator "+ + "must provide it") { + t.Fatalf("an absent access is not refused as before: %v", err) + } + for id, mode := range map[string]string{"radarr.library": "read-write", "sab.spool": "read-write"} { + o := outcomeOf(report, id) + if o.Action != "unchanged" || o.Detail != "operator-owned; present, "+mode+", nothing managed" { + t.Errorf("%s: %+v, and before it was unchanged, %q", id, o, "operator-owned; present, "+mode+", nothing managed") + } + } + if rootEmpty != "" { + if o := outcomeOf(report, "web.root"); o.Action != "unchanged" { + t.Errorf("a root-owned folder was judged: %+v", o) + } + } + if !started { + t.Error("a container mounting an absent access was held, and before it was created") + } + if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) { + t.Error("an absent access was made") + } + for _, d := range []string{unwritable, empty} { + if n, _ := unix.Getxattr(d, aclAccessXattr, nil); n > 0 { + t.Errorf("an ACL was set on %s", d) + } + if info := mustLstat(t, d); info.Mode().Perm() != 0o755 { + t.Errorf("%s changed mode: %o", d, info.Mode().Perm()) + } + } + if all, _ := KeptFolders.All(); len(all) != 0 { + t.Errorf("folders were recorded as kept: %v", all) + } + if len(Leftovers()) != 0 || len(WaitingOnAccess(declare(t, strings.Join(resources, ",")))) != 0 { + t.Error("something was said or held for an older controller's access") + } +} + +func mustLstat(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + return info +} + +// Review (low): a folder no access names, whose filesystem could not be named this look, is not said as its disk +// being away. +func TestALeftoverSaysWhyItsWalkCouldNotRun(t *testing.T) { + uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1" + _, lib, access := poolNamed(t, &uuid, &fsid, &source) + takeIt(t, lib, access) + other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(filepath.Dir(lib)), "x") + `"}` + if _, _, err := applyAccesses(t, store.State{}, other); err != nil { + t.Fatal(err) + } + uuid, fsid, source = "", "0000000000000000", "" // nothing names it now + ReconcileKept(context.Background()) + left := Leftovers() + if len(left) != 1 || strings.Contains(left[0].Why, "disk") || !strings.Contains(left[0].Why, "could not be named") { + t.Fatalf("the leftover does not say why its walk could not run: %+v", left) + } +} From d12bba01090de3106882eb8f3eb60ad490f48f53 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:34:59 +0200 Subject: [PATCH 10/10] Give the kept-folder tests' folders an unprivileged owner when they run as root (hq ADR 0325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build seat runs merge-check as root, so every folder a test made was root's data, which is never kept (§12), and 28 tests failed for that alone. A test's folders now stand for an operator's: run as root, they are given to nobody before they are judged. The root rule is unchanged. --- cmd/mesh-host/kept_test.go | 8 ++++ internal/apply/kept_test.go | 79 ++++++++++++++++++++++++++++++++- internal/apply/oldshape_test.go | 4 +- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/kept_test.go b/cmd/mesh-host/kept_test.go index 6c9c5dc..209ce37 100644 --- a/cmd/mesh-host/kept_test.go +++ b/cmd/mesh-host/kept_test.go @@ -33,6 +33,14 @@ func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) { if err := os.MkdirAll(filepath.Join(lib, "a film"), 0o755); err != nil { t.Fatal(err) } + // Run as root (the build seat's merge-check), the test's folders stand for an operator's, never root's. + if os.Geteuid() == 0 { + for _, p := range []string{dir, lib, filepath.Join(lib, "a film")} { + if err := os.Lchown(p, 65534, 65534); err != nil { + t.Fatal(err) + } + } + } v := judgeAccess(judgedAccess{module: "radarr", id: "radarr.movies", name: "movies", path: lib, users: []store.KeptUser{{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: "container"}}}) if v.state != link.StateWaiting || len(v.waits) != 1 { diff --git a/internal/apply/kept_test.go b/internal/apply/kept_test.go index 326be71..107f8ad 100644 --- a/internal/apply/kept_test.go +++ b/internal/apply/kept_test.go @@ -32,7 +32,7 @@ const ( // aclFolder is a folder of the test's own on a filesystem that carries ACLs, or the test is skipped saying why. func aclFolder(t *testing.T) string { t.Helper() - dir := t.TempDir() + dir := userDir(t) fd, err := unix.Open(dir, unix.O_RDONLY|unix.O_DIRECTORY, 0) if err != nil { t.Fatal(err) @@ -110,7 +110,9 @@ func effectiveOn(t *testing.T, path string, uid uint32) uint16 { func applyAccesses(t *testing.T, known store.State, resources ...string) (Report, store.State, error) { t.Helper() + toUser(t) d := declare(t, strings.Join(resources, ",")) + toUser(t) return Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, noServices, nil, nil) } @@ -135,6 +137,7 @@ func TestAKeptFolderCarriesAnEntryPerUserAndNothingElseOfItChanges(t *testing.T) t.Fatal(err) } unix.Close(fd) + toUser(t) _, _, before := readACLs(t, lib) report, _, err := applyAccesses(t, store.State{}, @@ -253,6 +256,7 @@ func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { if err := os.Chmod(usual, 0o644); err != nil { t.Fatal(err) } + toUser(t) c, err := JudgeKept(lib, rec.Want, ActLimits) if err != nil { t.Fatal(err) @@ -269,6 +273,7 @@ func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { if r, _ := keptRecord(lib); !r.OutOfWatches { t.Error("watches that ran out are not recorded") } + toUser(t) look := LookKept(lib, rec.Want, SweepLimits) if look.Repaired == 0 { t.Errorf("the sweep put nothing back: %+v", look) @@ -276,6 +281,7 @@ func TestWhatIsMadeInAKeptFolderIsGivenItsUsersRights(t *testing.T) { if got := effectiveOn(t, usual, radarr); got&6 != 6 { t.Fatalf("after the sweep a read-write identity may %s", permWords(got)) } + toUser(t) if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { t.Fatalf("after the sweep %d entries still give a user less than its right", c.Narrowed) } @@ -329,6 +335,7 @@ func TestAFolderThatHoldsDataIsTakenOnlyOnAWarrantAndLetGoEntryByEntry(t *testin } d := declare(t, a) + toUser(t) done, said, err := TakeFolder(context.Background(), d, "lidarr.music", lib, "warrant-7", "the operator", noServices) if err != nil { t.Fatalf("the take act was refused: %v", err) @@ -357,6 +364,7 @@ func TestAFolderThatHoldsDataIsTakenOnlyOnAWarrantAndLetGoEntryByEntry(t *testin t.Fatal(err) } beforeMask, _ := func() (uint16, bool) { a, _, _ := readACLs(t, song); return a.perm(aclMask, 0) }() + toUser(t) ReconcileKept(context.Background()) for _, p := range []string{lib, song} { a, def, _ := readACLs(t, p) @@ -397,6 +405,7 @@ func TestAWalkEntersNoOtherFilesystem(t *testing.T) { }, fd: unix.Fstat, } + toUser(t) w, err := walkKept(dir, ActLimits, stats, nil, keepVisit(map[uint32]bool{radarr: true})) if err != nil { t.Fatal(err) @@ -456,10 +465,12 @@ func TestAMissingFolderIsMadeWhereItsDataLivesAndWaitsWhereItsDiskIsNot(t *testi if _, err := os.Stat(books); !errors.Is(err, os.ErrNotExist) { t.Fatal("a folder was made where its disk is not") } + toUser(t) if v := AccessVerdict(context.Background(), books, false, true); v.Kind != VerdictWaitDisk { t.Errorf("the look judges it %s, not waiting for its disk", v.Kind) } // And the folder made before, its dataset gone: its disk is not there, whatever is at its path now. + toUser(t) if v := AccessVerdict(context.Background(), movies, false, true); v.Kind != VerdictWaitDisk { t.Errorf("a kept folder whose dataset is not mounted is judged %s", v.Kind) } @@ -492,12 +503,14 @@ func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing if err := os.Chmod(lib, 0o755); err != nil { t.Fatal(err) } + toUser(t) if short, _ := judgeTop(lib, rec.Want); len(short) == 0 { t.Fatal("a folder whose mask an app narrowed is not said short") } if _, _, err := applyAccesses(t, store.State{}, a); err != nil { t.Fatal(err) } + toUser(t) if short, _ := judgeTop(lib, rec.Want); len(short) != 0 { t.Fatalf("the apply did not put the folder back: %v", short) } @@ -506,6 +519,7 @@ func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing looks, repaired := 0, 0 for { looks++ + toUser(t) look := LookKept(lib, rec.Want, limits) repaired += look.Repaired if look.Failed != "" { @@ -527,6 +541,7 @@ func TestTheSweepPutsBackWhatAnAppNarrowedAndCoversTheFolderOverLooks(t *testing if repaired != 25 { t.Errorf("the sweep put back %d of 25 narrowed entries", repaired) } + toUser(t) if c, _ := JudgeKept(lib, rec.Want, ActLimits); c.Narrowed != 0 { t.Errorf("after the looks %d entries still give the user less than its right", c.Narrowed) } @@ -562,6 +577,7 @@ func TestRootsDataOnItsPlacedFilesystemIsNeverKept(t *testing.T) { if o := outcomeOf(report, "m.data"); o.Action == "waiting" { t.Errorf("root's data waits, and is never asked about: %+v", o) } + toUser(t) if v := AccessVerdict(context.Background(), path, false, true); v.Kind != VerdictRoot { t.Errorf("the look judges root's data at %s %s", path, v.Kind) } @@ -625,6 +641,7 @@ func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) { t.Fatal(err) } MountPoints = func() []string { return []string{"/"} } + toUser(t) ReconcileKept(context.Background()) rec, had := keptRecord(lib) if !had || len(rec.Users) != 2 { @@ -638,6 +655,7 @@ func TestNoWalkRunsWhereTheDiskIsNotThere(t *testing.T) { } // The disk back: the drop runs. MountPoints = func() []string { return []string{"/", dataset} } + toUser(t) ReconcileKept(context.Background()) if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, plex) }(); still { t.Fatal("the drop did not run once the disk was there") @@ -672,6 +690,7 @@ func TestADiskMountedOverAFolderKeptEmptyIsNotKeptWithoutAWarrant(t *testing.T) t.Fatal(err) } MountPoints = func() []string { return []string{"/", pool} } + toUser(t) if v := AccessVerdict(context.Background(), lib, false, true); v.Kind != VerdictWaitKeep { t.Fatalf("the look judges it %s, and the data on the new disk was never taken", v.Kind) } @@ -720,6 +739,7 @@ func TestWideningTheMaskGivesTheOwningGroupNothingMore(t *testing.T) { } before := groupRight() rec, _ := keptRecord(lib) + toUser(t) LookKept(lib, rec.Want, SweepLimits) if got := effectiveOn(t, ep, radarr); got&6 != 6 { t.Fatalf("the sweep did not widen the mask: %s", permWords(got)) @@ -825,13 +845,14 @@ func TestTheWatchEndsWithItsContext(t *testing.T) { // Review L6: a broken folder's words carry no path. func TestABrokenFoldersWordsCarryNoPath(t *testing.T) { - dir := t.TempDir() + dir := userDir(t) keeping(t, nil, nil) file := filepath.Join(dir, "notafolder") if err := os.WriteFile(file, nil, 0o644); err != nil { t.Fatal(err) } for _, path := range []string{file, filepath.Join(file, "below")} { + toUser(t) v := AccessVerdict(context.Background(), path, false, true) if v.Kind != VerdictBroken || strings.Contains(v.Why, "/") { t.Errorf("%s: %+v", path, v) @@ -896,6 +917,7 @@ func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) { } d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+ `,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`) + toUser(t) report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" { t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o) @@ -919,6 +941,7 @@ func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { t.Fatal(err) } MountPoints = func() []string { return []string{"/"} } + toUser(t) ReconcileKept(context.Background()) left := Leftovers() if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") || @@ -926,6 +949,7 @@ func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) { t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left) } MountPoints = func() []string { return []string{"/", dataset} } + toUser(t) ReconcileKept(context.Background()) if left := Leftovers(); len(left) != 0 { t.Fatalf("said after the drop ran: %+v", left) @@ -952,6 +976,7 @@ func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *te if _, _, err := applyAccesses(t, store.State{}, a); err != nil { t.Fatal(err) } + toUser(t) done, _, err := TakeFolder(context.Background(), declare(t, a), "radarr.media", lib, "w-1", "op", noServices) if err != nil { t.Fatal(err) @@ -971,6 +996,7 @@ func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *te } // The pool goes: the filesystem below holds the folder again, and the entries left there are taken out. MountPoints = func() []string { return []string{"/"} } + toUser(t) ReconcileKept(context.Background()) if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, filepath.Join(lib, "old")) @@ -1058,6 +1084,7 @@ func placedPool(t *testing.T, source, fsid *string) (pool, lib, taken string) { if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { t.Fatal(err) } + toUser(t) done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) if err != nil { t.Fatal(err) @@ -1082,6 +1109,7 @@ func TestAnotherFilesystemAtTheSameMountPointIsNotTaken(t *testing.T) { if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { t.Fatalf("another filesystem at the same mount point was kept on the old warrant: %+v", o) } + toUser(t) if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { t.Fatalf("the look judges it %s", v.Kind) } @@ -1100,6 +1128,7 @@ func TestADeviceRenamedIsTheSameFilesystem(t *testing.T) { if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { t.Fatalf("a device renamed reads as another filesystem: %+v", o) } + toUser(t) ReconcileKept(context.Background()) if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { t.Fatalf("the library's entries were walked away: the identity may %s", permWords(got)) @@ -1123,11 +1152,13 @@ func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil { t.Fatal(err) } + toUser(t) done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices) if err != nil { t.Fatal(err) } <-done + toUser(t) before := AccessVerdict(context.Background(), lib, true, true) film := filepath.Join(lib, "film") if err := os.WriteFile(film, []byte("data"), 0o644); err != nil { @@ -1148,10 +1179,12 @@ func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) { if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" { t.Fatalf("the second apply kept it on the old warrant: %+v", o) } + toUser(t) v := AccessVerdict(context.Background(), lib, true, true) if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") { t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before) } + toUser(t) ReconcileKept(context.Background()) if got := effectiveOn(t, film, radarr); got != 0 { t.Fatal("the data on the new filesystem was walked") @@ -1199,6 +1232,7 @@ func TestEntriesLeftOnTheFilesystemTheFolderIsOnAreNeverWalkedAway(t *testing.T) f.Left = []store.KeptLeft{{Mount: pool, Placed: f.Placed, Users: users}} return true }) + toUser(t) ReconcileKept(context.Background()) if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { t.Fatalf("kept=%v: entries on the folder's own filesystem were walked away", kept) @@ -1226,6 +1260,7 @@ func TestATakenOnOfTheOldFormIsNotTaken(t *testing.T) { f.Mount, f.TakenOn = "/", "/" return true }) + toUser(t) if v := AccessVerdict(context.Background(), lib, true, true); v.Kind != VerdictWaitKeep { t.Fatalf("an old-form record is read as taken: %s", v.Kind) } @@ -1263,6 +1298,7 @@ func takeIt(t *testing.T, lib, access string) { if _, _, err := applyAccesses(t, store.State{}, access); err != nil { t.Fatal(err) } + toUser(t) done, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w-1", "op", noServices) if err != nil { t.Fatal(err) @@ -1307,6 +1343,7 @@ func TestAFilesystemThatCannotBeNamedIsNeverKept(t *testing.T) { if got := effectiveOn(t, lib, radarr); got != 0 { t.Fatal("an entry was added on a filesystem that cannot be named") } + toUser(t) if _, _, err := TakeFolder(context.Background(), declare(t, access), "radarr.media", lib, "w", "op", noServices); err == nil { t.Fatal("the take act ran on a filesystem that cannot be named") } @@ -1327,6 +1364,7 @@ func TestAFilesystemsUUIDNamesItBeforeItsFsid(t *testing.T) { if o := outcomeOf(report, "radarr.media"); o.Action == "waiting" { t.Fatalf("a renumbered device with one UUID reads as another filesystem: %+v", o) } + toUser(t) ReconcileKept(context.Background()) if got := effectiveOn(t, filepath.Join(lib, "old"), radarr); got != 7 { t.Fatalf("the library's entries were walked away: %s", permWords(got)) @@ -1348,6 +1386,7 @@ func TestEntriesLeftOnAFilesystemThatNeverReturnsAreForgotten(t *testing.T) { if len(Leftovers()) != 1 { t.Fatal("the leftover is not said before it expires") } + toUser(t) said := ReconcileKept(context.Background()) if left := Leftovers(); len(left) != 0 { t.Fatalf("a leftover on a filesystem gone for longer than %s is still said: %+v", LeftFor, left) @@ -1446,3 +1485,39 @@ func flightsDone(t *testing.T) { return len(inFlight.reads) == 0 }) } + +// userDirs are the folders the running test made, which toUser gives an unprivileged owner when it runs as root. +var userDirs []string + +// userDir is a t.TempDir that toUser gives an unprivileged owner when the test runs as root (the build seat runs +// merge-check as root): root's data is never kept (ADR 0325 §12), and a test's own folders stand for an operator's. +func userDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + userDirs = append(userDirs, dir) + t.Cleanup(func() { userDirs = nil }) + return dir +} + +// unprivileged is the owner toUser gives a test's folders: nobody. +const unprivileged = 65534 + +// toUser gives everything under the running test's own folders to an unprivileged owner when the test runs as +// root, so they are judged as an operator's data is; as anyone else it changes nothing. +func toUser(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + return + } + for _, dir := range userDirs { + err := filepath.WalkDir(dir, func(p string, _ os.DirEntry, err error) error { + if err != nil { + return err + } + return os.Lchown(p, unprivileged, unprivileged) + }) + if err != nil { + t.Fatalf("giving the test's folders to an unprivileged owner: %v", err) + } + } +} diff --git a/internal/apply/oldshape_test.go b/internal/apply/oldshape_test.go index a9d23da..a4eb521 100644 --- a/internal/apply/oldshape_test.go +++ b/internal/apply/oldshape_test.go @@ -22,7 +22,7 @@ import ( // made, no container held, nothing judged. These are origin/main's outcomes, word for word. func TestAnOlderControllersAccessesAreAppliedExactlyAsBefore(t *testing.T) { keeping(t, nil, nil) - dir := t.TempDir() + dir := userDir(t) unwritable := filepath.Join(dir, "library") // its group may not write it empty := filepath.Join(dir, "spool") absent := filepath.Join(dir, "media", "movies") @@ -66,6 +66,7 @@ func TestAnOlderControllersAccessesAreAppliedExactlyAsBefore(t *testing.T) { } return "", nil } + toUser(t) report, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")), store.State{}, store.OriginDeclared, run, nil, nil) @@ -126,6 +127,7 @@ func TestALeftoverSaysWhyItsWalkCouldNotRun(t *testing.T) { t.Fatal(err) } uuid, fsid, source = "", "0000000000000000", "" // nothing names it now + toUser(t) ReconcileKept(context.Background()) left := Leftovers() if len(left) != 1 || strings.Contains(left[0].Why, "disk") || !strings.Contains(left[0].Why, "could not be named") {