novox/mesh-host
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request 'Give every module an identity and keep shared folders through ACL entries (hq ADR 0325, issue 567)' (#93) from feat/325-identities-and-kept-folders into main

Browse Source
This commit was merged in pull request #93.
This commit is contained in:
mesh-admin
2026-10-12 17:50:59 +00:00
parent dd9ab7927d d12bba0109
commit ef26dba275
39 changed files with 6854 additions and 1615 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/grants.go
-181
View File
@@ -1,181 +0,0 @@
package main
import (
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/link"
)
// A read-write access whose group cannot write is never quiet (novox/hq ADR 0312 §2).
//
// The engine grants a read-write access's path group to the identities the module's code runs as, and changes
// nothing on the path: whether its group may write is the operator's, as the data is. So an access whose path's
// group has no write permission is stated, on every look, as its module's resource of kind `access`, unhealthy,
// saying so and that the operator decides — and one owned by a group that is never granted (§4) is stated
// unhealthy saying that. The controller raises it as that module's condition on two statements in a row and
// clears it on the first healthy one, as it does every resource's health (ADR 0240). The path is the target,
// never in the reason: the controller says a reason on the operator's channel, which withholds a path (ADR 0234
// §6).
// grantedAccess is one read-write access that grants, as the judge reads it.
type grantedAccess struct {
module, id, path string
}
// accessJudge reads, on every look, whether each read-write access that grants may write its path through its
// group. Safe for the apply and the looking loop at once.
type accessJudge struct {
mu sync.Mutex
accesses []grantedAccess
// was is each access's last verdict, with since when and how many looks in a row found it unhealthy.
was map[string]link.ResourceHealth
// reading guards inFlight: every path whose read has not returned yet.
reading sync.Mutex
inFlight map[string]bool
}
// accesses is the one access judge on this machine.
var accesses = newAccessJudge()
func newAccessJudge() *accessJudge {
return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
}
// set is what the declaration just applied asks: every read-write access of a module that names a grant, not
// held as found. An access no longer declared is forgotten.
func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) {
var as []grantedAccess
if d != nil {
for _, r := range d.Resources {
a, ok := r.(*declaration.Access)
if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite || held[a.ID] {
continue
}
at := strings.LastIndex(a.ID, ".")
if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
continue
}
as = append(as, grantedAccess{module: a.ID[:at], id: a.ID, path: a.Path})
}
}
sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id })
j.mu.Lock()
defer j.mu.Unlock()
j.accesses = as
declared := map[string]bool{}
for _, a := range as {
declared[a.id] = true
}
for id := range j.was {
if !declared[id] {
delete(j.was, id)
}
}
}
// lookBound is how long a look waits for one accessed path to answer. A hard network mount whose server is down
// blocks a stat for ever, and one access must never stop every health statement this machine makes.
var lookBound = 5 * time.Second
// pathGroupBounded reads a path's group within lookBound, outside any lock. A read that has not answered is said
// so; while it still hangs, later looks do not start another on the same path, so a dead mount costs one blocked
// read, not one per look.
func (j *accessJudge) pathGroupBounded(path string) (apply.PathGroup, bool, error) {
j.reading.Lock()
if j.inFlight[path] {
j.reading.Unlock()
return apply.PathGroup{}, false, nil
}
j.inFlight[path] = true
j.reading.Unlock()
type read struct {
g apply.PathGroup
err error
}
done := make(chan read, 1)
go func() {
g, err := apply.GroupOfPath(path)
j.reading.Lock()
delete(j.inFlight, path)
j.reading.Unlock()
done <- read{g, err}
}()
select {
case r := <-done:
return r.g, true, r.err
case <-time.After(lookBound):
return apply.PathGroup{}, false, nil
}
}
// look reads every access now: healthy when its group may write and may be granted, unhealthy saying why
// otherwise. Whether any verdict changed since the last look, for the loop to say it. The paths are read with no
// lock held, each within lookBound.
func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
j.mu.Lock()
as := append([]grantedAccess(nil), j.accesses...)
j.mu.Unlock()
type verdict struct{ state, reason string }
read := make([]verdict, len(as))
for i, a := range as {
state, reason := link.StateHealthy, ""
g, answered, err := j.pathGroupBounded(a.path)
switch {
case !answered:
state, reason = link.StateUnhealthy, "its path does not answer: reading which group owns it did not "+
"return within "+lookBound.String()
case err != nil:
state, reason = link.StateUnhealthy, "its group cannot be read: "+strings.ReplaceAll(err.Error(), a.path, "the path")
case apply.GrantRefusal(g) != nil:
state, reason = link.StateUnhealthy, "its group is never granted: "+apply.GrantRefusal(g).Error()
case !g.Writable:
state, reason = link.StateUnhealthy, apply.CannotWriteWords(g)
}
read[i] = verdict{state, reason}
}
j.mu.Lock()
defer j.mu.Unlock()
changed := false
out := make([]link.ResourceHealth, 0, len(as))
for i, a := range as {
state, reason := read[i].state, read[i].reason
v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
State: state, Reason: reason, Since: now.UTC()}
if was, had := j.was[a.id]; had && was.State == state && was.Reason == reason {
v.Since = was.Since
v.Streak = was.Streak
} else {
changed = true
}
if state != link.StateHealthy {
v.Streak++
}
j.was[a.id] = v
out = append(out, v)
}
return out, changed
}
// withAccesses adds to a statement every read-write access that grants, as its module's resource of kind access.
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
h.Resources = append(h.Resources, vs...)
return h
}
// accessesHealthy says no access in vs is anything but healthy.
func accessesHealthy(vs []link.ResourceHealth) bool {
for _, v := range vs {
if v.State != link.StateHealthy {
return false
}
}
return true
}
cmd/mesh-host/grants_test.go
-138
View File
@@ -1,138 +0,0 @@
package main
import (
"strings"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/link"
)
// A read-write access whose path's group cannot write is stated unhealthy on every look, saying so and that the
// operator decides, and one whose group is never granted says that (novox/hq ADR 0312 §2, §4). The path is the
// target and never in the reason (ADR 0234 §6); a read access, granting nothing, is not stated at all.
func TestAnAccessWhoseGroupCannotWriteIsStatedUnhealthyWithTheOperatorsRemedy(t *testing.T) {
spool, library, shared, read := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir()
groups := map[string]apply.PathGroup{
spool: {GID: 1002, Name: "downloads", Writable: false},
library: {GID: 1001, Name: "media", Writable: true},
shared: {GID: 0, Name: "root", Writable: true},
}
was := apply.GroupOfPath
apply.GroupOfPath = func(path string) (apply.PathGroup, error) { return groups[path], nil }
t.Cleanup(func() { apply.GroupOfPath = was })
wasRoot := apply.RootGroupIDs
apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
t.Cleanup(func() { apply.RootGroupIDs = wasRoot })
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
{"id":"nzbget.downloads","type":"access","path":"` + spool + `","mode":"read-write","grant":{"account":"operator"}},
{"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{"account":"operator"}},
{"id":"importer.shared","type":"access","path":"` + shared + `","mode":"read-write","grant":{"account":"operator"}},
{"id":"player.music","type":"access","path":"` + read + `","mode":"read"}]}`))
if err != nil {
t.Fatal(err)
}
j := newAccessJudge()
j.set(d, nil)
at := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
vs, changed := j.look(at)
if !changed || len(vs) != 3 {
t.Fatalf("stated %d accesses (changed %v), want the three read-write ones: %+v", len(vs), changed, vs)
}
got := map[string]link.ResourceHealth{}
for _, v := range vs {
got[v.Resource] = v
if v.Kind != link.KindAccess || strings.Contains(v.Reason, v.Target) {
t.Errorf("%s is stated as %q with its path in the reason %q", v.Resource, v.Kind, v.Reason)
}
}
if v := got["nzbget.downloads"]; v.State != link.StateUnhealthy || v.Module != "nzbget" || v.Target != spool ||
!strings.Contains(v.Reason, "cannot write") || !strings.Contains(v.Reason, "operator's to decide") || v.Streak != 1 {
t.Errorf("an access whose group cannot write is stated %+v", v)
}
if v := got["importer.shared"]; v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "never granted") {
t.Errorf("an access of the root group is stated %+v", v)
}
if v := got["importer.music"]; v.State != link.StateHealthy {
t.Errorf("an access whose group may write is stated %+v", v)
}
if h := withAccesses(&link.Health{}, vs); accessesHealthy(vs) || len(h.Resources) != 3 {
t.Errorf("the statement does not carry the accesses as unhealthy: %+v", h.Resources)
}
// Looked at again: the same verdict, its since kept and its streak counted.
vs, changed = j.look(at.Add(time.Minute))
for _, v := range vs {
if v.Resource == "nzbget.downloads" && (changed || v.Streak != 2 || !v.Since.Equal(at)) {
t.Errorf("the second look is %+v (changed %v)", v, changed)
}
}
// The operator gives the group write: healthy at the next look, and nothing the engine did made it so.
groups[spool] = apply.PathGroup{GID: 1002, Name: "downloads", Writable: true}
vs, _ = j.look(at.Add(2 * time.Minute))
for _, v := range vs {
if v.Resource == "nzbget.downloads" && v.State != link.StateHealthy {
t.Errorf("an access whose group was given write is stated %+v", v)
}
}
}
// A path that does not answer — a hard network mount whose server is down — is stated unhealthy within the bound,
// and stops neither this look nor the next: one blocked read per path, never one per look.
func TestAnAccessWhosePathDoesNotAnswerIsStatedWithinTheBound(t *testing.T) {
hung, library := t.TempDir(), t.TempDir()
release, finished := make(chan struct{}), make(chan struct{})
var started atomic.Int32
was, wasBound := apply.GroupOfPath, lookBound
apply.GroupOfPath = func(path string) (apply.PathGroup, error) {
if path == hung {
started.Add(1)
<-release
defer close(finished)
}
return apply.PathGroup{GID: 1001, Name: "media", Writable: true}, nil
}
lookBound = 50 * time.Millisecond
wasRoot := apply.RootGroupIDs
apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
t.Cleanup(func() { apply.GroupOfPath, lookBound, apply.RootGroupIDs = was, wasBound, wasRoot })
// Run first: the read that hangs is let go and has returned before anything it read is put back.
t.Cleanup(func() { close(release); <-finished })
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
{"id":"nzbget.downloads","type":"access","path":"` + hung + `","mode":"read-write","grant":{}},
{"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{}}]}`))
if err != nil {
t.Fatal(err)
}
j := newAccessJudge()
j.set(d, nil)
for look := 0; look < 3; look++ {
done := make(chan []link.ResourceHealth, 1)
go func() { vs, _ := j.look(time.Now()); done <- vs }()
select {
case vs := <-done:
for _, v := range vs {
switch v.Resource {
case "nzbget.downloads":
if v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "does not answer") {
t.Errorf("look %d: a path that does not answer is stated %+v", look, v)
}
case "importer.music":
if v.State != link.StateHealthy {
t.Errorf("look %d: the path that answers is stated %+v", look, v)
}
}
}
case <-time.After(5 * time.Second):
t.Fatalf("look %d did not return while one path hangs", look)
}
}
if n := started.Load(); n != 1 {
t.Errorf("%d reads were started on the path that hangs; want one", n)
}
}
cmd/mesh-host/kept.go
+431
View File
@@ -0,0 +1,431 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// An access is judged by what each of its users may do (novox/hq ADR 0325 §7, §10, §12).
//
// On every look the engine judges each module's access as its module's resource of kind `access`: waiting, with a
// wait of kind access, while the mesh may not keep its folder yet or its disk is not there; unhealthy for root's
// data on its placed filesystem, and for a kept folder that gives a user less than its right — read by entry and
// mask together, on the folder and the sweep's sample under it, which the sweep puts back where it can; healthy
// otherwise. The folder is the target, never in a reason nor in a wait's words: the controller says them on the
// operator's channel, which withholds a path (ADR 0234 §6).
// judgedAccess is one access as the judge reads it.
type judgedAccess struct {
module, id, name, path string
taken bool
users []store.KeptUser
// leftover is the mesh's entries nobody wants, not an access (link.KindKeptEntries).
leftover bool
}
// accessJudge reads every access on every look. Safe for the apply and the looking loop at once.
type accessJudge struct {
mu sync.Mutex
accesses []judgedAccess
// was is each access's last verdict, with since when and how many looks in a row found it unhealthy.
was map[string]link.ResourceHealth
// reading guards inFlight: every folder whose look has not returned yet.
reading sync.Mutex
inFlight map[string]bool
}
// accesses is the one access judge on this machine.
var accesses = newAccessJudge()
func newAccessJudge() *accessJudge {
return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
}
// set is what the declaration just applied asks: every access of a module, not held as found, with the users of
// its folder as every access of that folder gives them. An access no longer declared is forgotten.
func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) {
var as []judgedAccess
if d != nil {
users := map[string]map[uint32]store.KeptUser{}
taken := map[string]bool{}
for _, r := range d.Resources {
if a, ok := r.(*declaration.Access); ok {
path := filepath.Clean(a.Path)
if users[path] == nil {
users[path] = map[uint32]store.KeptUser{}
}
taken[path] = taken[path] || a.Taken
for _, u := range a.Users {
had, seen := users[path][uint32(u.UID)]
if !seen || (u.Use == declaration.AccessReadWrite && had.Use != declaration.AccessReadWrite) {
users[path][uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID),
Use: u.Use, From: u.From}
}
}
}
}
for _, r := range d.Resources {
a, ok := r.(*declaration.Access)
if !ok || held[a.ID] || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
continue
}
// An older controller's access is judged by nothing, as before ADR 0325: no state is said of it.
if !apply.Keeps(d, a.Path) {
continue
}
at := strings.LastIndex(a.ID, ".")
if at <= 0 {
continue
}
path := filepath.Clean(a.Path)
ja := judgedAccess{module: a.ID[:at], id: a.ID, name: a.ID[at+1:], path: path, taken: taken[path]}
for _, u := range users[path] {
ja.users = append(ja.users, u)
}
sort.Slice(ja.users, func(x, y int) bool { return ja.users[x].UID < ja.users[y].UID })
as = append(as, ja)
}
}
sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id })
j.mu.Lock()
defer j.mu.Unlock()
j.accesses = as
declared := map[string]bool{}
for _, a := range as {
declared[a.id] = true
}
for id := range j.was {
if !declared[id] {
delete(j.was, id)
}
}
}
// lookBound is how long a look waits for one accessed folder to answer. A hard network mount whose server is down
// blocks a stat for ever, and one access must never stop every health statement this machine makes.
var lookBound = 5 * time.Second
// verdict is one access's judgement in a look.
type verdict struct {
state, reason string
waits []link.AccessWait
act *link.AccessActRecord
entries []link.KeptEntry
}
// judgeBounded judges one access within lookBound, outside any lock. A look that has not answered is said so;
// while it still hangs, later looks start no other on the same folder, so a dead mount costs one blocked read.
func (j *accessJudge) judgeBounded(a judgedAccess) (verdict, bool) {
j.reading.Lock()
if j.inFlight[a.path] {
j.reading.Unlock()
return verdict{}, false
}
j.inFlight[a.path] = true
j.reading.Unlock()
done := make(chan verdict, 1)
go func() {
v := judgeAccess(a)
j.reading.Lock()
delete(j.inFlight, a.path)
j.reading.Unlock()
done <- v
}()
select {
case v := <-done:
return v, true
case <-time.After(lookBound):
return verdict{}, false
}
}
// judgeAccess is one access's judgement now (ADR 0325 §7, §10, §12), the sweep of its kept folder included.
func judgeAccess(a judgedAccess) verdict {
v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0)
switch v.Kind {
case apply.VerdictWaitKeep:
return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken on the filesystem " +
"it is on now, and waits for the operator's word (novox/hq ADR 0325 §7, §8)",
waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
What: apply.KeepWords(a.name), Placed: v.Placed}}}
case apply.VerdictWaitDisk:
return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)",
waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
What: apply.DiskWords(a.name)}}}
case apply.VerdictRoot:
return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)}
case apply.VerdictUnknown:
// Not a fault and not the operator's: said as not known, and judged again on the next look.
return verdict{state: link.StateUnknown, reason: v.Why + "; judged again on the next look"}
case apply.VerdictBroken:
// Its words carry no path (AccessVerdict says them so): the reason reaches the operator's channel.
return verdict{state: link.StateUnhealthy, reason: v.Why}
case apply.VerdictMake:
return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " +
"its data lives (novox/hq ADR 0325 §9)"}
case apply.VerdictNobody:
return verdict{state: link.StateHealthy}
}
out := verdict{state: link.StateHealthy, entries: entriesOf(a.users)}
rec, _ := keptNow(a.path)
if act := rec.Act; act != nil {
out.act = &link.AccessActRecord{Act: act.Act, Access: a.id, Warrant: act.Warrant, By: act.By, State: act.State,
Started: act.Started, Ended: act.Ended, Folders: act.Folders, Files: act.Files, Kept: act.Kept,
Links: act.Links, Other: act.Other, Mounts: act.Mounts, Why: act.Why}
}
if !rec.Kept {
out.state, out.reason = link.StateStarting, "the controller says it was taken; the next apply keeps it"
return out
}
look := apply.LookKept(a.path, a.users, apply.SweepLimits)
var faults []string
if look.Failed != "" {
faults = append(faults, "the sweep could not look: "+apply.Pathless(errors.New(look.Failed)))
}
if len(look.Short) > 0 {
faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+
"apply")
}
if n := look.Sweep.Unrepaired + look.Sweep.LastUnrepaired; n > 0 {
faults = append(faults, fmt.Sprintf("%d entries under it give a user less than its right and cannot be "+
"put back: an owner's own bits, which the mesh never changes, or an entry the filesystem refused", n))
}
if len(faults) > 0 {
out.state, out.reason = link.StateUnhealthy, strings.Join(faults, "; ")
return out
}
switch {
case rec.Act != nil && rec.Act.State == link.ActRunning:
out.state, out.reason = link.StateStarting, "the take act is adding its users' entries under it"
case !sameUsersList(rec.Users, rec.Want):
out.state, out.reason = link.StateStarting, "the entries under it are being changed to its users now"
}
return out
}
// entriesOf is a kept folder's listing (ADR 0325 §4): each user, and beside the tool runner's what it stands for.
func entriesOf(users []store.KeptUser) []link.KeptEntry {
var out []link.KeptEntry
for _, u := range users {
e := link.KeptEntry{Identity: u.Identity, UID: int(u.UID), Use: u.Use, From: u.From}
if u.From == declaration.FromToolRunner {
e.Note = "the tool runner's account stands for every module whose tools use this folder, at the widest " +
"use any of them declares: a module that declared only read may write through it (novox/hq ADR " +
"0325 §4, until issue 569)"
}
out = append(out, e)
}
return out
}
func sameUsersList(a, b []store.KeptUser) bool {
if len(a) != len(b) {
return false
}
seen := map[uint32]string{}
for _, u := range a {
seen[u.UID] = u.Use
}
for _, u := range b {
if use, ok := seen[u.UID]; !ok || use != u.Use {
return false
}
}
return true
}
// keptNow is the record of one kept folder, as the engine's file holds it.
func keptNow(path string) (store.KeptFolder, bool) {
all, err := apply.KeptFolders.All()
if err != nil {
return store.KeptFolder{}, false
}
f, had := all[path]
return f, had
}
// look reads every access now. Whether any verdict changed since the last look, for the loop to say it.
func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
j.mu.Lock()
as := append([]judgedAccess(nil), j.accesses...)
j.mu.Unlock()
read := make([]verdict, len(as))
for i, a := range as {
v, answered := j.judgeBounded(a)
if !answered {
v = verdict{state: link.StateUnhealthy, reason: "its folder does not answer: judging it did not return " +
"within " + lookBound.String()}
}
read[i] = v
}
// And the entries nobody wants that could not be taken out yet, each under the access that last named its
// folder, until its drop walk has run (ADR 0325 §8).
for _, l := range apply.Leftovers() {
as = append(as, judgedAccess{module: l.Module, id: l.Access + "~entries", name: l.Name, path: l.Path,
leftover: true})
read = append(read, verdict{state: link.StateUnhealthy, reason: l.Why})
}
j.mu.Lock()
defer j.mu.Unlock()
changed := false
out := make([]link.ResourceHealth, 0, len(as))
for i, a := range as {
r := read[i]
kind := link.KindAccess
if a.leftover {
kind = link.KindKeptEntries
}
v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: kind, Target: a.path,
State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries}
if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason {
v.Since = was.Since
v.Streak = was.Streak
} else {
changed = true
}
if r.state == link.StateUnhealthy {
v.Streak++
}
j.was[a.id] = v
out = append(out, v)
}
return out, changed
}
// withAccesses adds to a statement every access, as its module's resource of kind access.
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
h.Resources = append(h.Resources, vs...)
return h
}
// accessesHealthy says no access in vs is anything but healthy.
func accessesHealthy(vs []link.ResourceHealth) bool {
for _, v := range vs {
if v.State != link.StateHealthy {
return false
}
}
return true
}
// keeper runs the walks under kept folders beside the applies, one at a time (ADR 0325 §8): poked after every
// apply and every look, it walks whatever folder's entries under it are not what was declared.
type keeper struct {
running, following atomic.Bool
say func(string)
watch *apply.Watcher
}
// keeping is the one keeper on this machine.
var keeping = &keeper{say: func(string) {}}
// poke starts a reconcile of the kept folders unless one is running, and has the watch follow what is kept.
func (k *keeper) poke(ctx context.Context) {
// The watch is set beside, never in the caller's way: watching a new folder's tree reads every folder of it.
if k.following.CompareAndSwap(false, true) {
go func() {
defer k.following.Store(false)
k.follow()
}()
}
if !k.running.CompareAndSwap(false, true) {
return
}
go func() {
defer k.running.Store(false)
for _, line := range apply.ReconcileKept(ctx) {
k.say(line)
}
}()
}
// follow has the watch watch every kept folder with its users.
func (k *keeper) follow() {
if k.watch == nil {
return
}
all, err := apply.KeptFolders.All()
if err != nil {
return
}
folders := map[string][]store.KeptUser{}
for path, f := range all {
if f.Kept && len(f.Want) > 0 {
folders[path] = f.Want
}
}
k.watch.Set(folders)
}
// startKeeping opens the watch and has the keeper say what it does. A machine without the watch is said so, and
// its sweep alone finds what is made.
func startKeeping(ctx context.Context, say func(string)) {
keeping.say = say
w, err := apply.NewWatcher()
if err != nil {
say(fmt.Sprintf("the kept folders are not watched (%v): the sweep alone finds what is made in them", err))
} else {
keeping.watch = w
go w.Run(ctx)
}
keeping.poke(ctx)
}
// accessActFrom is the take act as the controller asks it over the link (ADR 0325 §8): judged against the
// declaration this node keeps, and run beside the applies.
func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
return func(act link.AccessAct) (string, error) {
kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
if errors.Is(err, store.ErrNothingDeclared) {
return "", errors.New("this node keeps no declaration, so it is declared no access")
}
if err != nil {
return "", err
}
d, err := declaration.Parse(kept.Declaration)
if err != nil {
return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
_, said, err := apply.TakeFolder(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By,
apply.ExecRunner)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
return "", err
}
return said, nil
}
}
// withWaiting is held with every id of waiting beside it: what liveness does not judge.
func withWaiting(held, waiting map[string]bool) map[string]bool {
if len(waiting) == 0 {
return held
}
out := map[string]bool{}
for id := range held {
out[id] = true
}
for id := range waiting {
out[id] = true
}
return out
}
cmd/mesh-host/kept_test.go
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// Row 4: the listing of a kept folder says, beside the tool runner's entry, that it stands for every module whose
// tools use the folder at the widest use (novox/hq ADR 0325 §4).
func TestTheListingSaysWhatTheToolRunnersEntryStandsFor(t *testing.T) {
entries := entriesOf([]store.KeptUser{
{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: declaration.FromContainer},
{Identity: "nox", UID: 61003, Use: "read-write", From: declaration.FromToolRunner},
})
if entries[0].Note != "" || !strings.Contains(entries[1].Note, "widest use") {
t.Fatalf("the listing: %+v", entries)
}
}
// Row 7, the engine's part: a folder that holds data and was never taken is said waiting with one wait of kind
// access — the access by its module's own id, the part its module, and words with no path (novox/hq ADR 0325 §7).
func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) {
dir := t.TempDir()
lib := filepath.Join(dir, "movies")
if err := os.MkdirAll(filepath.Join(lib, "a film"), 0o755); err != nil {
t.Fatal(err)
}
// Run as root (the build seat's merge-check), the test's folders stand for an operator's, never root's.
if os.Geteuid() == 0 {
for _, p := range []string{dir, lib, filepath.Join(lib, "a film")} {
if err := os.Lchown(p, 65534, 65534); err != nil {
t.Fatal(err)
}
}
}
v := judgeAccess(judgedAccess{module: "radarr", id: "radarr.movies", name: "movies", path: lib,
users: []store.KeptUser{{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: "container"}}})
if v.state != link.StateWaiting || len(v.waits) != 1 {
t.Fatalf("not waiting with one wait: %+v", v)
}
raw, _ := json.Marshal(v.waits[0])
want := `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies",` +
`"placed":"` + v.waits[0].Placed + `"}`
if string(raw) != want || !strings.HasPrefix(v.waits[0].Placed, "fs-") {
t.Fatalf("the wait reads %s", raw)
}
if strings.Contains(v.reason, dir) {
t.Fatalf("the reason names the path: %s", v.reason)
}
}
// An older controller's accesses (no users, no identities) are judged by nothing, as before ADR 0325: the statement
// carries no resource for them, waiting, unknown or kept-entries.
func TestAnOlderControllersAccessesSayNothing(t *testing.T) {
dir := t.TempDir()
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
{"id":"radarr.library","type":"access","path":"` + filepath.Join(dir, "library") + `","mode":"read-write"},
{"id":"plex.movies","type":"access","path":"` + filepath.Join(dir, "absent") + `"}]}`))
if err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(dir, "library"), 0o755); err != nil {
t.Fatal(err)
}
j := newAccessJudge()
j.set(d, nil)
if vs, _ := j.look(time.Now()); len(vs) != 0 {
t.Fatalf("an older controller's accesses are judged: %+v", vs)
}
}
cmd/mesh-host/main.go
+23 -12
View File
@@ -241,6 +241,8 @@ func run(ctx context.Context, command string, opts options) error {
// found (novox/hq issue 339).
handedOverFrom(opts.state)
foundState.Store(opts.state)
// And what it knows of the shared folders it keeps is beside its state (novox/hq ADR 0325 §5).
apply.KeptFolders = store.NewKeptFile(opts.state)
// And the guard asks the running container runtimes where they keep their data (novox/hq issue 339).
apply.AskRuntimes = apply.ExecRunner
switch command {
@@ -1121,6 +1123,8 @@ func runLink(ctx context.Context, opts options) error {
// this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224).
sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
go sched.Run(ctx)
// The watch on every kept folder and the walks under them, beside the applies (novox/hq ADR 0325 §6, §8).
startKeeping(ctx, func(line string) { say(line) })
// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
// report this host makes carries what it says.
@@ -1232,8 +1236,10 @@ func runLink(ctx context.Context, opts options) error {
// A read-only judgement of the directories this node is declared, asked by the controller (novox/hq
// issue 358): which the walk that follows no link another account could make would refuse.
JudgeDirectories: judgeDirectoriesFrom(opts.state),
Say: say,
Timeout: opts.timeout,
// The take act on an access's folder, on the operator's warrant (novox/hq ADR 0325 §8).
AccessAct: accessActFrom(opts.state, opts.timeout),
Say: say,
Timeout: opts.timeout,
}
worker := make(chan struct{})
go func() {
@@ -1596,7 +1602,8 @@ const (
// judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
// rule 6).
// rule 6) — but for the sweep of a kept folder, which ADR 0325 §10 has every look make: it puts back an entry or a
// mask an app narrowed, and nothing else.
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
ticker := time.NewTicker(liveness.LookEvery)
defer ticker.Stop()
@@ -1635,8 +1642,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
}
}
// Every read-write access that grants, read on every look (novox/hq ADR 0312 §2).
// Every access, judged on every look, its kept folder swept (novox/hq ADR 0325 §10); and a walk stopped
// at its bound goes on.
accessSt, accessesChanged := accesses.look(time.Now())
keeping.poke(ctx)
owed = owed || accessesChanged
if accessesChanged {
for _, v := range accessSt {
@@ -1691,10 +1700,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
// healthAsReported is a statement as the report and the event carry it.
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
// GrantContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
// user's declared `root` (novox/hq ADR 0266), and grants a read-write access's path group (novox/hq ADR 0312),
// which is what tells the controller it may be sent any of those fields.
h := &link.Health{Contract: link.GrantContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
// IdentityContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, a container's identity and
// an access's users (novox/hq ADR 0325), which is what tells the controller it may be sent any of those fields.
h := &link.Health{Contract: link.IdentityContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
for _, r := range st.Resources {
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
@@ -2075,7 +2084,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
// applied, so every resource this apply started says `starting` and the gate waits for it.
if j := judging; j != nil {
j.Set(liveness.LongRunning(declared, held))
// A container that waits for an accessed folder not there was not created: the access says why, and
// liveness judges it neither alive nor dead (novox/hq ADR 0325 §9).
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnAccess(declared))))
st, _ := j.Look(ctx)
netJudge.declare(declaredResolvConf(declared))
var netSt *network.Statement
@@ -2093,13 +2104,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
// said with it, relogin needed included, in the report that says the apply.
var accountSt *accounts.Statement
if a := accountJudge; a != nil {
// A read-write access's account too (novox/hq ADR 0312 §6): relogin needed is said for a session of
// the account's own manager that began before the grant, as for a user resource's group.
a.Set(append(accounts.Of(declared, held), apply.GrantedAccounts(declared, held)...))
a.Set(accounts.Of(declared, held))
as, _ := a.Look(ctx)
accountSt = &as
}
accesses.set(declared, held)
// What the apply changed in who uses a kept folder is walked under it now, beside (ADR 0325 §8).
keeping.poke(ctx)
accessSt, _ := accesses.look(time.Now())
report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
accountSt), foundDirectories(updated)), accessSt)
cmd/mesh-host/units_test.go
+4 -2
View File
@@ -87,8 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
}}
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
// At RootContract or later: an engine that also grants an access's group says GrantContract (novox/hq ADR 0312).
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.GrantContract {
// At RootContract or later: an engine that also reads identities and keeps shared folders says IdentityContract
// (novox/hq ADR 0325), one above ADR 0312's GrantContract.
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.IdentityContract ||
link.IdentityContract != link.GrantContract+1 {
t.Fatalf("contract %d", h.Contract)
}
got := map[string]string{}
internal/apply/access_test.go
-23
View File
@@ -3,7 +3,6 @@ package apply
import (
"context"
"os"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
@@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) {
}
}
// Absent is refused, not created. A bind mount whose source does not exist is made by the
// container runtime as root, with whatever mode it picks — the silent wrong-ownership
// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather
// than conjuring a directory it does not own.
func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) {
missing := t.TempDir() + "/media/library" // named, never created
d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`)
_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginDeclared, noServices, nil, nil)
if err == nil {
t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime")
}
if !strings.Contains(err.Error(), "the operator must provide") ||
!strings.Contains(err.Error(), "does not own") {
t.Fatalf("the refusal does not say whose the path is: %v", err)
}
if _, statErr := os.Stat(missing); statErr == nil {
t.Fatal("the host created the path it does not own")
}
}
// Undeclaring an access never removes the path. Unassigning the module that reached the media
// library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a
// directory the mesh never made. The record is dropped; the operator's data is left exactly as it
internal/apply/acl.go
+242
View File
@@ -0,0 +1,242 @@
package apply
// The POSIX ACL a kept folder carries, as the kernel stores it (novox/hq ADR 0325 §5, issue 567).
//
// **Read and written as the kernel's own extended attribute, never through setfacl.** Linux keeps a file's access
// ACL in the attribute system.posix_acl_access and a folder's default ACL in system.posix_acl_default, each a
// little-endian header (version 2) followed by entries of a tag, a permission and an id. The engine reads and
// writes them on a descriptor it opened itself (fgetxattr, fsetxattr), so a name swapped for a link between the
// look and the change changes nothing, and no program is run per file of a library of millions.
//
// **What the mesh changes is narrow.** It adds or sets one named-user entry per uid it keeps, widens the mask to
// what the entries need, and on a file with no ACL yet writes the owning group's entry as exactly the group bits
// the mode had: owner, owning group, the owner's bits, others' bits and every other entry stay as found.
import (
"encoding/binary"
"errors"
"fmt"
"sort"
)
// The attributes and the tags of the kernel's ACL format (linux/posix_acl_xattr.h).
const (
aclAccessXattr = "system.posix_acl_access"
aclDefaultXattr = "system.posix_acl_default"
aclVersion = 2
aclUserObj = 0x01
aclUser = 0x02
aclGroupObj = 0x04
aclGroup = 0x08
aclMask = 0x10
aclOther = 0x20
// aclUndefinedID is the id of an entry that names nobody: the owner, the owning group, the mask, others.
aclUndefinedID = 0xFFFFFFFF
)
// The permission bits of one entry.
const (
permRead = 4
permWrite = 2
permExec = 1
)
// aclEntry is one entry: its tag, its permission bits (rwx as 4, 2, 1) and, for a named user or group, its id.
type aclEntry struct {
Tag uint16
Perm uint16
ID uint32
}
// acl is a whole ACL as the kernel holds it.
type acl []aclEntry
// decodeACL reads an attribute's bytes. An attribute that is not version 2, or not whole entries, is refused.
func decodeACL(raw []byte) (acl, error) {
if len(raw) < 4 || (len(raw)-4)%8 != 0 {
return nil, fmt.Errorf("an ACL attribute of %d bytes is not a header and whole entries", len(raw))
}
if v := binary.LittleEndian.Uint32(raw); v != aclVersion {
return nil, fmt.Errorf("an ACL attribute of version %d, and the engine reads version %d", v, aclVersion)
}
out := make(acl, 0, (len(raw)-4)/8)
for at := 4; at < len(raw); at += 8 {
out = append(out, aclEntry{
Tag: binary.LittleEndian.Uint16(raw[at:]),
Perm: binary.LittleEndian.Uint16(raw[at+2:]),
ID: binary.LittleEndian.Uint32(raw[at+4:]),
})
}
return out, nil
}
// encode is the attribute's bytes, the entries in the order the kernel requires: by tag, then by id.
func (a acl) encode() []byte {
sorted := append(acl(nil), a...)
sort.SliceStable(sorted, func(i, j int) bool {
if sorted[i].Tag != sorted[j].Tag {
return sorted[i].Tag < sorted[j].Tag
}
return sorted[i].ID < sorted[j].ID
})
out := make([]byte, 4, 4+8*len(sorted))
binary.LittleEndian.PutUint32(out, aclVersion)
for _, e := range sorted {
var b [8]byte
binary.LittleEndian.PutUint16(b[0:], e.Tag)
binary.LittleEndian.PutUint16(b[2:], e.Perm)
binary.LittleEndian.PutUint32(b[4:], e.ID)
out = append(out, b[:]...)
}
return out
}
// minimalACL is the ACL a mode alone stands for: the owner's, the owning group's and others' bits.
func minimalACL(mode uint32) acl {
return acl{
{Tag: aclUserObj, Perm: uint16(mode>>6) & 7, ID: aclUndefinedID},
{Tag: aclGroupObj, Perm: uint16(mode>>3) & 7, ID: aclUndefinedID},
{Tag: aclOther, Perm: uint16(mode) & 7, ID: aclUndefinedID},
}
}
// find is the index of the entry with tag and id, or -1.
func (a acl) find(tag uint16, id uint32) int {
for i, e := range a {
if e.Tag == tag && (e.ID == id || (tag != aclUser && tag != aclGroup)) {
return i
}
}
return -1
}
// perm is the bits of the entry with tag and id, and whether there is one.
func (a acl) perm(tag uint16, id uint32) (uint16, bool) {
if i := a.find(tag, id); i >= 0 {
return a[i].Perm, true
}
return 0, false
}
// check refuses an ACL the kernel would: one owner's, owning group's and others' entry each, and a mask whenever
// a named entry is there.
func (a acl) check() error {
count := map[uint16]int{}
named := false
for _, e := range a {
count[e.Tag]++
if e.Tag == aclUser || e.Tag == aclGroup {
named = true
}
}
if count[aclUserObj] != 1 || count[aclGroupObj] != 1 || count[aclOther] != 1 || count[aclMask] > 1 {
return errors.New("an ACL without exactly one owner's, owning group's and others' entry")
}
if named && count[aclMask] != 1 {
return errors.New("an ACL with named entries and no mask")
}
return nil
}
// keptACL is a with the mesh's entries made: for each uid of need its entry set to exactly its bits, and the mask
// widened by what those entries need — never narrowed, so an app's wider mask stays. A mask already there keeps
// what it gave every other entry: one holding a bit the widening adds loses that bit alone; with none yet (a file the mode alone described) the mask starts at the owning
// group's bits, which minimalACL gave its entry, so the owning group keeps exactly the right its group bits gave.
// Nothing else is changed.
func keptACL(a acl, need map[uint32]uint16) acl {
out := append(acl(nil), a...)
var widen uint16
for uid, bits := range need {
widen |= bits
if i := out.find(aclUser, uid); i >= 0 {
out[i].Perm = bits
} else {
out = append(out, aclEntry{Tag: aclUser, Perm: bits, ID: uid})
}
}
if i := out.find(aclMask, 0); i >= 0 {
old := out[i].Perm
out[i].Perm |= widen
if added := out[i].Perm &^ old; added != 0 {
// **A wider mask widens nobody else.** The owning group's entry and every named entry the mesh does
// not set here were limited by the old mask: a file made at 0644 under a default ACL carries the
// owning group's rwx with a mask of r--. One mask serves them all, so an entry holding a bit the mask
// now adds cannot keep its effective right without losing that bit: exactly those bits are taken off
// it, and nothing else of it changes. An entry holding none of them is not touched at all.
for j := range out {
e := &out[j]
_, ours := need[e.ID]
if (e.Tag == aclGroupObj || e.Tag == aclGroup || (e.Tag == aclUser && !ours)) && e.Perm&added != 0 {
e.Perm &^= added
}
}
}
return out
}
if len(need) == 0 {
return out
}
group, _ := out.perm(aclGroupObj, 0)
return append(out, aclEntry{Tag: aclMask, Perm: group | widen, ID: aclUndefinedID})
}
// withoutUsers is a with the named-user entry of each uid of drop taken out, and nothing else changed: the mask
// stays as it was, so no other entry loses a right.
func withoutUsers(a acl, drop map[uint32]bool) acl {
out := make(acl, 0, len(a))
for _, e := range a {
if e.Tag == aclUser && drop[e.ID] {
continue
}
out = append(out, e)
}
return out
}
// effective is the right a uid has through a, given the file's owner and mode: the owner's bits for the owner,
// else its named entry limited by the mask, else nothing a named entry gives (the owning group's and others'
// rights are not the mesh's to count on). An ACL with no mask limits nothing.
func effective(a acl, uid, owner uint32, mode uint32) uint16 {
if uid == owner {
return uint16(mode>>6) & 7
}
bits, ok := a.perm(aclUser, uid)
if !ok {
return 0
}
if mask, has := a.perm(aclMask, 0); has {
bits &= mask
}
return bits
}
// needOn is what a use needs on a folder (read and enter; with read-write, write too) or on a file (read; with
// read-write, write): entering means nothing on a file, and an execute bit given to data would be a right nobody
// asked for.
func needOn(readWrite, folder bool) uint16 {
bits := uint16(permRead)
if readWrite {
bits |= permWrite
}
if folder {
bits |= permExec
}
return bits
}
// permWords is bits as ls writes them: "rwx", "r-x".
func permWords(bits uint16) string {
out := []byte("---")
if bits&permRead != 0 {
out[0] = 'r'
}
if bits&permWrite != 0 {
out[1] = 'w'
}
if bits&permExec != 0 {
out[2] = 'x'
}
return string(out)
}
internal/apply/apply.go
+131 -75
View File
@@ -262,6 +262,12 @@ func ApplyMindingWindows(
}
report = Report{}
// **No module identity is root** (novox/hq ADR 0325 §11): an identity or a container told a root number refuses
// the declaration whole, before anything of it is made or started.
if err := refuseRootIdentities(d); err != nil {
return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report}
}
declared := map[string]bool{}
for _, r := range d.Resources {
declared[r.Identity()] = true
@@ -315,7 +321,7 @@ func ApplyMindingWindows(
// (novox/hq ADR 0252), on the unit's rule above: two modules may each want the account in one group,
// and the one going takes nothing the other still needs.
wanted := groupsWanted(d.Resources)
// And what a record going hands to a resource that still asks for its group (novox/hq ADR 0312 §5).
// And what a record going hands to a resource that still asks for its group (novox/hq ADR 0252 §3).
handed := Handed{}
removeOrphan := func(orphan store.Applied) error {
@@ -343,8 +349,25 @@ func ApplyMindingWindows(
case declaration.TypeUser:
action, detail, err = removeUser(ctx, sys, orphan, run, wanted, handed)
case declaration.TypeAccess:
// The path is left as it is; what it granted an account is taken back (novox/hq ADR 0312 §5).
action, detail = removeAccess(ctx, sys, orphan, run, wanted, handed)
// The folder is left as it is: what the mesh added to it is taken out by the walk once no access names
// it (novox/hq ADR 0325 §8), never by removing the access's record.
action, detail = "forgotten", "an operator-owned path is never the host's to remove"
if _, kept := keptRecord(filepath.Clean(orphan.Target)); kept {
detail += "; the entries the mesh added go once no access names it"
}
gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed)
if said != "" {
detail = joinDetail(detail, said)
if gave {
action = "restored"
}
}
if len(left) > 0 {
// Kept recorded with what is still to take back, and tried again (ADR 0177's waiting removal).
orphan.Groups = left
known.Record(orphan)
err = fmt.Errorf("%w: %s", errRemovalWaits, said)
}
default:
action, detail, err = remove(ctx, sys, orphan, run, made)
}
@@ -516,12 +539,11 @@ func ApplyMindingWindows(
// which declared directories were already here: one this apply makes is the mesh's, one found may not be.
refreshRuntimeRoots(ctx)
in.refused = refusedPlaces(d.Resources)
// The group of every read-write access that grants, read once (novox/hq ADR 0312): what each container is
// created with, and what each account is asked to be in beside its user resources' groups — read before any
// orphan goes, so one access going leaves an account in a group another still grants.
in.grants = grantsOf(d.Resources, in.refused)
in.containerGroups = containerGroups(d.Resources, in.grants)
wanted.withGrants(d.Resources, in.grants)
// What every declared access asks of its folder, several accesses of one folder as one (novox/hq ADR 0325 §5),
// and every accessed folder, which no container mounts while it is not there (§9).
in.folders = foldersOf(d.Resources, in.refused)
in.identities = len(d.Identities) > 0
in.accesses = accessPaths(d.Resources, in.refused, in.identities)
in.agentHomes = rootNeverHomes(d)
in.dirsBefore = map[string]bool{}
for _, resource := range d.Resources {
@@ -563,6 +585,34 @@ func ApplyMindingWindows(
sort.Strings(out)
return out
}
// **Every module's identity is made before anything of it is applied** (novox/hq ADR 0325 §3): a directory
// it owns is given to it, and a container is told to run as it. One that cannot be made — its name or numbers
// another's here — fails, naming whose, and its module is not attempted; the rest of the machine is.
var identityFailures []*Error
if len(d.Identities) > 0 {
tool, toolErr := toolFor(sys)
for _, id := range d.Identities {
var outcome Outcome
err := toolErr
if err == nil {
outcome, err = applyIdentity(ctx, tool, id, &known, run)
}
if err != nil {
// Counted with the others once the resources are through: here, before the guard, it would read
// as the guard not being up.
identityFailures = append(identityFailures, &Error{Resource: "identity " + id.Name, Err: err})
log(fmt.Sprintf(" failed identity %s: %v", id.Name, err))
gated[id.Module] = true
gatedWhy[id.Module] = "its identity " + id.Name + " could not be made, so nothing of this module " +
"was attempted"
continue
}
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action != "unchanged" {
log(fmt.Sprintf(" %s %s: %s", outcome.Action, outcome.ID, outcome.Detail))
}
}
}
for i, resource := range ordered {
if !orphansRemoved && i == guardFirst {
// **Only a guard that is up may let the filter go.** Removing the derived filter's
@@ -925,6 +975,8 @@ func ApplyMindingWindows(
}
}
failures = append(failures, identityFailures...)
// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
// what it replaces running and recorded, said, for the next apply to hand over.
unhanded := make([]string, 0, len(handover))
@@ -945,6 +997,19 @@ func ApplyMindingWindows(
}
}
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
if origin == store.OriginDeclared {
// A folder no access names any more has what the mesh added taken out by the next walk (ADR 0325 §8).
forgetUndeclared(in.folders)
// And an identity no longer declared goes, once what ran as it went above (ADR 0325 §3).
if tool, err := toolFor(sys); err == nil {
for _, o := range removeIdentities(ctx, tool, d.Identities, &known, run) {
report.Outcomes = append(report.Outcomes, o)
log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail))
}
}
}
// A converged node whose found firewall was in force retires it only now, once everything —
// the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100), and on every
// converged apply, not once (ADR 0168). Skipped, it is said: a step that does nothing is never
@@ -1072,6 +1137,14 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
if err := refuseMounts(res, in.refused); err != nil {
return begin(r), err
}
// Not created over an accessed folder not there, which the runtime would make as root (novox/hq ADR 0325 §9).
if id := mountsMissing(ctx, res, in.accesses); id != "" {
out := begin(r)
out.Action = "waiting"
out.Detail = "not created: it mounts the access " + id + ", whose folder or its disk is not there; " +
"created once it is (novox/hq ADR 0325 §9)"
return out, nil
}
}
// Nothing below a home is touched through a link an account put there (novox/hq ADR 0266).
// And nothing is unpacked or written into below an agent's home (agent_home_guard.go).
@@ -1304,45 +1377,6 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O
// library, a download spool that several modules share — and the host's only job is to be sure it
// is there before anything mounts it.
//
// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by
// the container runtime, as root, with whatever mode it picks — which is exactly the silent
// wrong-ownership 04-ISSUES/026 records. So the host checks first and says plainly that the
// operator must provide the path, rather than conjuring a directory it does not own and cannot
// give the right owner. Nothing is written, so this never reports a change: the machine did not
// move, the host merely confirmed a fact about it.
//
// **A read-write access grants its path's group** (novox/hq ADR 0312): read, never changed — grantAccess.
func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
in inputs) (Outcome, error) {
out := begin(r)
info, err := os.Stat(r.Path)
if errors.Is(err, os.ErrNotExist) {
return out, fmt.Errorf(
"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
"nothing, so a missing one is said here rather than made as root by the container "+
"runtime", r.Path)
}
if err != nil {
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf(
"%s is not a directory, and an access is a shared directory the operator provides",
r.Path)
}
mode := declaration.AccessRead
if r.Mode != "" {
mode = r.Mode
}
out.Action = "unchanged"
out.Detail = "operator-owned; present, " + mode + ", nothing managed"
if err := grantAccess(ctx, sys, r, run, previous, in, &out); err != nil {
return out, err
}
return out, nil
}
// keepFound, when not nil, is where the original of a file this host has no record of is kept
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
@@ -2160,10 +2194,12 @@ type inputs struct {
// dirsBefore is every declared directory that was on the machine when this apply began (novox/hq issue
// 339): one the apply itself made — a file's parent — is the mesh's, one that was there may not be.
dirsBefore map[string]bool
// grants is what every read-write access that grants grants, by access id, read once (novox/hq ADR 0312);
// containerGroups the number of every group granted to a container, by container id.
grants map[string]grant
containerGroups map[string][]int
// folders is what every declared access asks of its folder, by path (novox/hq ADR 0325 §5); accesses every
// accessed folder with its access's id, which no container mounts while it is not there (§9).
folders map[string]*folderWant
// identities is that the declaration carries identities: its controller speaks ADR 0325 for every access.
identities bool
accesses map[string]string
// handed is every group a record that went this apply handed to a resource still asking for it.
handed Handed
}
@@ -2240,15 +2276,13 @@ func (in inputs) reads(r *declaration.Container) map[string]string {
// containerSpec is the identity of a declared container: everything that, if changed, means
// the running container is no longer what was asked for.
func containerSpec(r *declaration.Container, in inputs) string {
return containerSpecReading(r, in.declares, in.reads(r), in.containerGroups[r.ID])
return containerSpecReading(r, in.declares, in.reads(r))
}
// containerSpecReading is containerSpec with what the container reads already read — so an
// applier that also records those digests reads each file once, and the label and the record
// cannot disagree about a file that moved between two reads.
//
// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312).
func containerSpecReading(r *declaration.Container, declares, reads map[string]string, groups []int) string {
func containerSpecReading(r *declaration.Container, declares, reads map[string]string) string {
keys := make([]string, 0, len(r.Env))
for k := range r.Env {
keys = append(keys, k)
@@ -2327,11 +2361,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
if h := r.Health; h.RunByRuntime() {
b.WriteString("health " + strings.Join(runtimeCheckArgs(h), " ") + "\n")
}
// And the groups its read-write accesses grant it (novox/hq ADR 0312 §6): a process's supplementary groups
// are fixed when the container is created, so one gained, dropped or renumbered is a container recreated
// with it. Only when there is one, so no container that is granted nothing has its digest moved.
for _, gid := range groups {
b.WriteString("group " + strconv.Itoa(gid) + "\n")
// And who it runs as (novox/hq ADR 0325 §2, §3): fixed when the container is created, so an identity told
// another way, or renumbered, is a container recreated with it. Only when there is one, so no container that
// runs as its image says has its digest moved.
if args := identityArgs(r); len(args) > 0 {
b.WriteString("identity " + strings.Join(args, " ") + "\n")
}
// **What this container reads is part of what it is.**
//
@@ -2433,8 +2467,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
out := begin(r)
// Read once, so the spec and the record agree on what was read even if a file moves under them.
reads := in.reads(r)
groups := in.containerGroups[r.ID]
want := containerSpecReading(r, in.declares, reads, groups)
want := containerSpecReading(r, in.declares, reads)
out.reads = reads
cri, err := containerRuntime(ctx, run)
@@ -2462,7 +2495,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
}
if r.RunOnce {
return applyRunOnce(ctx, r, run, cri, want, groups, previous)
return applyRunOnce(ctx, r, run, cri, want, previous)
}
before, err := containerState(ctx, r.Name, run)
@@ -2507,7 +2540,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
// genuine recreate. The trade-off, stated: a container that was ALREADY stale when the host
// upgraded — created against a file that has since changed — is not caught by this, and could
// not be by the alternative either, which recreates it without knowing whether it needed to.
legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil, groups) &&
legacy := len(reads) > 0 && before.Spec == containerSpecReading(r, in.declares, nil) &&
(wasReading == nil || sameReads(wasReading, reads))
// **A container a maintenance window holds still is left as it is** (novox/hq issue 224, ADR
@@ -2577,7 +2610,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
for _, v := range r.Volumes {
args = append(args, "--volume", v)
}
args = append(args, groupAddArgs(groups)...)
args = append(args, identityArgs(r)...)
// **The one check the container carries is the one the module declared** (ADR 0240 rule 3): an
// exec command, or the image's own adopted by name, with the declared timing. Nothing else on the
// machine sets a container's check.
@@ -2634,9 +2667,9 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
}
case len(reasons) > 0:
out.Detail = "recreated to pick up " + strings.Join(reasons, ", ")
case len(groups) > 0 && before.Spec == containerSpecReading(r, in.declares, reads, nil):
// It was created before its read-write accesses granted it anything (novox/hq ADR 0312 §6).
out.Detail = "recreated to hold the groups its read-write accesses grant (gid " + gidWords(groups) + ")"
case r.RunAs != nil && before.Spec == specWithoutIdentity(r, in.declares, reads):
// It was created before it was told its module's identity (novox/hq ADR 0325 §3).
out.Detail = "recreated to run as its module's identity (novox/hq ADR 0325 §3)"
default:
out.Detail = "replaced; a container's configuration is fixed when it is created"
}
@@ -2676,7 +2709,7 @@ func runtimeCheckArgs(h *declaration.Health) []string {
// - It writes a seed only if it is absent and never reconciles it, so what a running program
// grows in that seed afterward is never wiped (04-ISSUES/035). The host's marker keeps the
// step from re-running; the step's own code keeps it from clobbering on the pass it does run.
func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, groups []int,
func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string,
previous store.Applied) (Outcome, error) {
out := begin(r)
@@ -2696,7 +2729,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri
// Run in the foreground so the runtime waits for the container and hands back its exit code.
// No --detach and no --restart: a step that is restarted is not a step.
args := foregroundRunArgs(r, want, groups)
args := foregroundRunArgs(r, want)
if _, err := run(ctx, cri, args...); err != nil {
// A non-zero exit or a runtime that could not start it. Either way the step did not make
@@ -2718,9 +2751,7 @@ func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri
// exit code — no --detach, no --restart, because a step that is restarted is not a step. Shared by
// a run-once step (novox/hq ADR 0052) and by one fire of a scheduled step (ADR 0053), which are the
// same "run the container and let it exit" up to how often it happens.
//
// groups is the number of every group its read-write accesses grant it (novox/hq ADR 0312).
func foregroundRunArgs(r *declaration.Container, want string, groups []int) []string {
func foregroundRunArgs(r *declaration.Container, want string) []string {
args := []string{"run", "--name", r.Name}
for _, file := range r.EnvFile {
args = append(args, "--env-file", file)
@@ -2735,7 +2766,7 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st
for _, v := range r.Volumes {
args = append(args, "--volume", v)
}
args = append(args, groupAddArgs(groups)...)
args = append(args, identityArgs(r)...)
for _, h := range r.Hosts {
args = append(args, "--add-host", h)
}
@@ -2744,6 +2775,31 @@ func foregroundRunArgs(r *declaration.Container, want string, groups []int) []st
return args
}
// specWithoutIdentity is the spec r had before it was told who to run as.
func specWithoutIdentity(r *declaration.Container, declares, reads map[string]string) string {
plain := *r
plain.RunAs = nil
return containerSpecReading(&plain, declares, reads)
}
// identityArgs is the runtime's arguments that run a container as its module's identity (novox/hq ADR 0325 §2):
// the image told by the variables it reads, or by the runtime's own user option, by number — a container's own
// account database is not the machine's. None for a container that runs as its image says.
func identityArgs(r *declaration.Container) []string {
id := r.RunAs
if id == nil {
return nil
}
if id.Mode == declaration.IdentityByRun {
return []string{"--user", strconv.Itoa(id.UID) + ":" + strconv.Itoa(id.GID)}
}
args := []string{"--env", id.UserEnv + "=" + strconv.Itoa(id.UID)}
if id.GroupEnv != "" {
args = append(args, "--env", id.GroupEnv+"="+strconv.Itoa(id.GID))
}
return args
}
// ensureImage makes the pinned image present on the node without running anything.
//
// A service or a run-once container gets its image as a side effect of `docker run` — the first run
internal/apply/grants.go
-514
View File
@@ -1,514 +0,0 @@
package apply
import (
"context"
"fmt"
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// A read-write access grants the path's own group (novox/hq ADR 0312, issue 537).
//
// **An access is not owning.** ADR 0051 left the operator's shared paths exactly as they are: the engine
// confirms one is there and mounts it, and sets no owner, group or mode on it or below it. Nothing decided how
// a `read-write` access then becomes writable, and a module whose identity was not the path's owner could not
// write it at all — a download client could not remove a folder it had filed (issue 535).
//
// **The path's group is the operator's statement of who may share it.** So the engine reads, at apply, which
// group owns the accessed path — only the machine knows — and makes the identities the controller names for
// the access members of it: each container that mounts it gains the group as a supplementary group of its
// process (`--group-add`, by number, because a container's own group database is not the machine's), and the
// account that runs the module's tools is put in it as an account group (ADR 0252 §1). The module then writes
// the path as its group may, and as nothing else.
//
// **The path is never changed** (§2). A group that cannot write the path is the operator's choice, as the data
// is: the access grants nothing, is stated unhealthy on every look, saying so and that the operator decides, and
// nothing is done to make it writable. **A read access grants nothing** (§3). **The root group is never granted** (§4): a
// path owned by group 0, or by a group whose members act as root (accounts.RootGroups, the account judge's own
// closed list, judged by the numbers those groups have here), or by a number the machine names nothing for, is
// refused by name.
//
// **What was granted is taken back as ADR 0252 takes back** (§5): the access records each group it put the
// account in that the account was not in before; when no declared resource asks for it any more, the account
// leaves it, and a group the account held before is never recorded and never taken back. A container needs no
// record: the group is part of what it is created as, so one no longer granted is recreated without it.
//
// **And what runs the code is restarted to hold it** (§6). A container is recreated with the group, because
// its spec carries it. The tool runner — a process of the machine's own service manager with `User=` — is
// restarted because the controller names the access under its `restart-on`, and the access is reported
// updated exactly when the account's groups moved: that manager reads the account's groups from the database
// each time it starts the unit, so the restarted process holds the group with no new login. A unit of the
// account's own manager, or a login session, keeps the groups its login began with: the account judge
// (internal/accounts) says *relogin needed* there, for the access's account as for a user resource's.
// PathGroup is the group that owns an accessed path, as the engine reads it.
type PathGroup struct {
GID int
// Name is the group's name in the machine's group database; empty when the database names nothing for
// the number.
Name string
// Writable is whether the group's write bit is set on the path itself.
Writable bool
}
// Words names the group for a person: its name and number, or its number alone.
func (g PathGroup) Words() string {
if g.Name == "" {
return fmt.Sprintf("gid %d", g.GID)
}
return fmt.Sprintf("%s (gid %d)", g.Name, g.GID)
}
// GroupOfPath reads which group owns a path, by its number, its name and whether it may write. Reads only. A
// variable so a test can stand in for a machine whose groups it cannot make.
var GroupOfPath = func(path string) (PathGroup, error) {
info, err := os.Stat(path)
if err != nil {
return PathGroup{}, err
}
_, gid, ok := statOwner(info)
if !ok {
return PathGroup{}, fmt.Errorf("the group that owns %s cannot be read on this machine", path)
}
g := PathGroup{GID: gid, Writable: info.Mode().Perm()&0o020 != 0}
if found, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil {
g.Name = found.Name
}
return g, nil
}
// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine
// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine
// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number
// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and
// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can
// stand in.
var RootGroupIDs = func() map[int]string {
rootIDs.Lock()
defer rootIDs.Unlock()
if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute {
rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now()
}
return rootIDs.ids
}
var rootIDs struct {
sync.Mutex
ids map[int]string
at time.Time
}
func readRootGroupIDs() map[int]string {
out := map[int]string{}
if raw, err := os.ReadFile("/etc/group"); err == nil {
for name, gid := range groupEntries(string(raw)) {
if slices.Contains(accounts.RootGroups, name) {
out[gid] = name
}
}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
for _, name := range accounts.RootGroups {
raw, err := exec.CommandContext(ctx, "getent", "group", name).Output()
if err != nil {
continue
}
for found, gid := range groupEntries(string(raw)) {
if found == name {
out[gid] = name
}
}
}
return out
}
// groupEntries reads group database lines, name:password:gid:members, as name to number.
func groupEntries(raw string) map[string]int {
out := map[string]int{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Split(strings.TrimSpace(line), ":")
if len(fields) < 3 || strings.HasPrefix(fields[0], "#") {
continue
}
if gid, err := strconv.Atoi(fields[2]); err == nil {
out[fields[0]] = gid
}
}
return out
}
// GrantRefusal is why a path's group is never granted (novox/hq ADR 0312 §4), or nil when it may be: group 0, a
// group whose members act as root by membership alone — judged by its NUMBER against every number those groups
// have here, and by its name — or a group this machine names nothing for, which cannot be judged at all.
func GrantRefusal(g PathGroup) error {
const never = "a shared path is never a way to root (novox/hq ADR 0312 §4); nothing was granted, and nothing " +
"was changed on it"
if g.GID == 0 {
return fmt.Errorf("it is owned by group 0 (%s), and a read-write access never grants the root group: %s",
orRoot(g.Name), never)
}
if root, is := RootGroupIDs()[g.GID]; is {
return fmt.Errorf("it is owned by gid %d, which is %s here, whose members act as root, and a read-write "+
"access never grants such a group: %s", g.GID, root, never)
}
if slices.Contains(accounts.RootGroups, g.Name) {
return fmt.Errorf("it is owned by the group %s, whose members act as root, and a read-write access never "+
"grants such a group: %s", g.Words(), never)
}
if g.Name == "" {
return fmt.Errorf("it is owned by gid %d, which this machine's group database names nothing, so whether "+
"its members act as root cannot be judged, and an unjudged group is never granted: %s", g.GID, never)
}
return nil
}
// CannotWriteWords is the reason an access whose group may not write its path is stated unhealthy with (novox/hq
// ADR 0312 §2). Never the path: the controller says a reason on the operator's channel, which withholds a path
// (ADR 0234 §6), and keeps the target as evidence.
func CannotWriteWords(g PathGroup) string {
return fmt.Sprintf("its group cannot write it: %s owns it without group write, and the mesh changes nothing on "+
"an accessed path (novox/hq ADR 0312 §2) — whether its group may write is the operator's to decide",
g.Words())
}
func orRoot(name string) string {
if name == "" {
return "root"
}
return name
}
// grant is what one read-write access grants, read once per apply.
type grant struct {
path string
group PathGroup
// refused is why nothing is granted: the path could not be read, or its group is never granted.
refused error
}
// grants says the group is given: it may be granted, and it may write the path. A group that cannot write is
// granted nothing (novox/hq ADR 0312 §2) — the access is stated unhealthy for the operator to decide.
func (g grant) grants() bool { return g.refused == nil && g.group.Writable }
// grantsOf reads the group of every read-write access that names a grant, by access id. One that is refused
// grants nothing — the access itself fails, saying why — and one whose path the engine refuses to place
// anything at (issue 339) is not read at all.
func grantsOf(resources []declaration.Resource, refused map[string]error) map[string]grant {
out := map[string]grant{}
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite {
continue
}
if why := refused[filepath.Clean(a.Path)]; why != nil {
out[a.ID] = grant{path: a.Path, refused: why}
continue
}
g, err := GroupOfPath(a.Path)
if err == nil {
err = GrantRefusal(g)
}
out[a.ID] = grant{path: a.Path, group: g, refused: err}
}
return out
}
// containerGroups is, for each container, the number of every group a read-write access grants it, sorted and
// once each: what it is created with as supplementary groups.
func containerGroups(resources []declaration.Resource, grants map[string]grant) map[string][]int {
out := map[string][]int{}
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok || a.Grant == nil {
continue
}
g, granted := grants[a.ID]
if !granted || !g.grants() {
continue
}
for _, id := range a.Grant.Containers {
if !slices.Contains(out[id], g.group.GID) {
out[id] = append(out[id], g.group.GID)
}
}
}
for id := range out {
sort.Ints(out[id])
}
return out
}
// ContainerGroupsOf is containerGroups for a whole declaration, read now: what the scheduler arms a scheduled
// step with, so a fire runs with the groups an apply would have created it with. A path issue 339's guard refuses
// is not read, as the apply reads none.
func ContainerGroupsOf(d *declaration.Declaration) map[string][]int {
if d == nil {
return nil
}
return containerGroups(d.Resources, grantsOf(d.Resources, refusedPlaces(d.Resources)))
}
// withGrants adds to what the declaration asks of each account the groups its read-write accesses grant (novox/hq
// ADR 0312 §5): one user resource or access going leaves an account in a group another still asks for.
func (w Wanted) withGrants(resources []declaration.Resource, grants map[string]grant) Wanted {
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok || a.Grant == nil || a.Grant.Account == "" {
continue
}
g, granted := grants[a.ID]
if !granted || !g.grants() {
continue
}
if w[a.Grant.Account] == nil {
w[a.Grant.Account] = map[string][]string{}
}
w[a.Grant.Account][g.group.Name] = append(w[a.Grant.Account][g.group.Name], a.ID)
}
return w
}
// GrantedAccounts is every account a read-write access puts in its path's group, as the account judge reads
// them (novox/hq ADR 0312 §6): judged like a user resource's account groups, so a session of the account's own
// manager that began before is said *relogin needed*, and only there. Read now; a path whose group is refused
// or cannot be read asks nothing of the judge — the access says that itself.
func GrantedAccounts(d *declaration.Declaration, held map[string]bool) []accounts.Account {
if d == nil {
return nil
}
grants := grantsOf(d.Resources, refusedPlaces(d.Resources))
var out []accounts.Account
for _, r := range d.Resources {
a, ok := r.(*declaration.Access)
if !ok || a.Grant == nil || a.Grant.Account == "" || held[a.ID] {
continue
}
g, granted := grants[a.ID]
if !granted || !g.grants() {
continue
}
at := strings.LastIndex(a.ID, ".")
if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
continue
}
out = append(out, accounts.Account{Module: a.ID[:at], ID: a.ID, Name: a.Grant.Account,
Groups: []string{g.group.Name}})
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
// grantAccess is applyAccess's grant (novox/hq ADR 0312): the account put in the path's group, what an earlier
// apply of this access put an account in and nobody asks for any more taken back, and the outcome said. The
// path itself is only read.
func grantAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
in inputs, out *Outcome) error {
var g grant
granting := r.Grant != nil && r.Mode == declaration.AccessReadWrite
if granting {
var read bool
g, read = in.grants[r.ID]
if !read {
// An apply that read nothing ahead — a caller with no inputs — reads it here.
g = grantsOf([]declaration.Resource{r}, in.refused)[r.ID]
}
if g.refused != nil {
return fmt.Errorf("%s grants nothing: %w", r.Path, g.refused)
}
}
// A group that cannot write is granted nothing (§2): what an earlier apply granted is taken back below.
account := ""
if granting && g.group.Writable {
account = r.Grant.Account
}
if account != "" && g.group.Name == "" {
return fmt.Errorf("%s is owned by %s, which this machine's group database names nothing, and an account "+
"is put in a group by name: %s was not (novox/hq ADR 0312 §1)", r.Path, g.group.Words(), account)
}
var said []string
moved := false
// What this access put an account in before. An access whose account changed gives the old one back
// everything it put it in; the new one starts with nothing recorded.
recorded := previous.Groups
if previous.User != "" && previous.User != account {
gave, words := takeBack(ctx, sys, previous.User, r.ID, previous.Groups, nil, run, in.groups)
moved = moved || gave
said = append(said, words...)
recorded = nil
}
var kept []string
if account != "" {
groups, err := system.GroupsOf(ctx, system.Runner(run), account)
if err != nil {
return err
}
inGroup := setOf(groups)[g.group.Name]
if !inGroup {
if err := sys.AddUserToGroup(ctx, system.Runner(run), account, g.group.Name); err != nil {
return err
}
moved = true
said = append(said, fmt.Sprintf("put %s in %s, the group that owns it; the tool runner holds it once "+
"restarted, a session that began before only after a new login", account, g.group.Name))
}
// Recorded only when the mesh put it there, now or before, or a record that went handed it over: a group
// held before is never taken back.
if !inGroup || slices.Contains(recorded, g.group.Name) || in.handed.has(account, g.group.Name) {
kept = append(kept, g.group.Name)
}
var others []string
for _, x := range recorded {
if x != g.group.Name {
others = append(others, x)
}
}
gave, words := takeBack(ctx, sys, account, r.ID, others, &kept, run, in.groups)
moved = moved || gave
said = append(said, words...)
}
if len(kept) > 0 {
out.groups, out.user = kept, account
}
if granting {
var to []string
if len(r.Grant.Containers) > 0 {
to = append(to, "its containers "+strings.Join(r.Grant.Containers, ", "))
}
if account != "" {
to = append(to, account)
}
switch {
case !g.group.Writable:
out.Detail = joinDetail(out.Detail, fmt.Sprintf("its group %s cannot write it, so nothing is granted: "+
"stated unhealthy, and nothing was changed on it — the operator decides", g.group.Words()))
case len(to) > 0:
out.Detail = joinDetail(out.Detail, fmt.Sprintf("grants %s to %s", g.group.Words(), strings.Join(to, " and ")))
}
}
if len(said) > 0 {
out.Detail = joinDetail(out.Detail, strings.Join(said, "; "))
}
if moved {
// Updated exactly when the account's groups moved: what names this access under restart-on — the
// tool runner — is restarted to hold them (novox/hq ADR 0312 §6).
out.Action = "updated"
}
return nil
}
// takeBack takes account out of every group of groups the mesh put it in for the resource id, unless another
// declared resource still asks for it — then it stays, and stays recorded on keep when keep is given, so it is
// taken back once nobody asks. A group it is no longer in is nothing to give back. Never fatal: a failure is
// said, and the group stays recorded where it can be. Whether any group was taken back, and what was said.
func takeBack(ctx context.Context, sys system.System, account, id string, groups []string, keep *[]string,
run Runner, wanted Wanted) (bool, []string) {
if len(groups) == 0 {
return false, nil
}
in, err := system.GroupsOf(ctx, system.Runner(run), account)
if err != nil {
if keep != nil {
*keep = append(*keep, groups...)
}
return false, []string{fmt.Sprintf("the groups the mesh put %s in (%s) were not given back: %v", account,
strings.Join(groups, ", "), err)}
}
already := setOf(in)
gave := false
var said []string
for _, g := range groups {
if !already[g] {
continue
}
if others := wanted.othersAsk(account, g, id); len(others) > 0 {
said = append(said, fmt.Sprintf("left %s in %s, which %s still asks for", account, g, strings.Join(others, ", ")))
if keep != nil {
*keep = append(*keep, g)
}
continue
}
gone, why := leaveGroup(ctx, sys, account, g, run)
if !gone && keep != nil {
*keep = append(*keep, g)
}
gave = gave || gone
said = append(said, account+" "+why)
}
return gave, said
}
// removeAccess is an access no longer declared: the path left exactly as it is (novox/hq ADR 0051), and every
// group it put an account in taken back unless another declared resource still asks for it (ADR 0312 §5) — then
// handed to that resource, which records it as the mesh's and takes it back when it stops asking.
func removeAccess(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted,
handed Handed) (string, string) {
const left = "an operator-owned path is never the host's to remove"
if a.User == "" || len(a.Groups) == 0 {
return "forgotten", left
}
var stays []string
gave, said := takeBack(ctx, sys, a.User, a.ID, a.Groups, &stays, run, wanted)
for _, g := range stays {
if len(wanted.othersAsk(a.User, g, a.ID)) > 0 {
handed.hand(a.User, g)
}
}
action := "forgotten"
if gave {
action = "restored"
}
return action, joinDetail(left, strings.Join(said, "; "))
}
// groupAddArgs is the runtime's arguments that give a container's process each granted group as a supplementary
// group, by number: the container's own group database is not the machine's, and a name could mean another
// group there, or none.
func groupAddArgs(groups []int) []string {
var args []string
for _, gid := range groups {
args = append(args, "--group-add", strconv.Itoa(gid))
}
return args
}
func gidWords(groups []int) string {
words := make([]string, 0, len(groups))
for _, gid := range groups {
words = append(words, strconv.Itoa(gid))
}
return strings.Join(words, ", ")
}
// Handed is every group a record that went in this apply put an account in, and another declared resource still
// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group
// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it
// there went first (novox/hq ADR 0252 §3, ADR 0312 §5).
type Handed map[string]map[string]bool
func (h Handed) hand(account, group string) {
if h == nil {
return
}
if h[account] == nil {
h[account] = map[string]bool{}
}
h[account][group] = true
}
func (h Handed) has(account, group string) bool { return h[account][group] }
internal/apply/grants_test.go
-559
View File
@@ -1,559 +0,0 @@
package apply
import (
"context"
"errors"
"fmt"
"os"
"os/user"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0312 and issue 537: a read-write access grants its path's own group to the containers and
// the account the controller names, and the path is never changed; a read access grants nothing; the root group,
// and a group whose members act as root, are refused by name; what was granted is taken back as ADR 0252 takes
// back; a container is recreated with the group, and a unit of the machine's own manager running as the account
// is restarted once the account holds it.
// grantMachine is the group database of groupDB, a container runtime that keeps each container's spec label and
// the arguments it was run with, and a service manager that notes, at each restart, the groups the account holds
// in the database then — which is what the machine's own manager gives the unit's new process.
type grantMachine struct {
*groupDB
containers map[string]string
runs map[string][]string
restarts []string
}
func newGrantMachine(account string, in []string, exists ...string) *grantMachine {
g := newGroupDB(in, exists...)
g.account = account
return &grantMachine{groupDB: g, containers: map[string]string{}, runs: map[string][]string{}}
}
func (m *grantMachine) run(ctx context.Context, name string, args ...string) (string, error) {
a := oldShape(name, args)
switch name {
case "docker":
m.asked = append(m.asked, name+" "+strings.Join(a, " "))
switch a[0] {
case "info":
return "27.0\n", nil
case "container":
spec, there := m.containers[a[len(a)-1]]
if !there {
return "", errors.New("no such container")
}
return "true\t" + spec, nil
case "rm":
delete(m.containers, a[len(a)-1])
case "run":
var named, spec string
for i := 0; i+1 < len(a); i++ {
switch {
case a[i] == "--name":
named = a[i+1]
case a[i] == "--label" && strings.HasPrefix(a[i+1], specLabel+"="):
spec = strings.TrimPrefix(a[i+1], specLabel+"=")
}
}
m.containers[named] = spec
m.runs[named] = a
return "deadbeef\n", nil
}
return "", nil
case "systemctl":
m.asked = append(m.asked, name+" "+strings.Join(a, " "))
if a[0] == "restart" {
m.restarts = append(m.restarts, a[1]+": "+m.groups())
}
return "", nil
}
return m.groupDB.run(ctx, name, args...)
}
// groupOfPaths stands in for the machine's group database and the paths' groups for the length of a test.
func groupOfPaths(t *testing.T, groups map[string]PathGroup) {
t.Helper()
was := GroupOfPath
GroupOfPath = func(path string) (PathGroup, error) {
if _, err := os.Stat(path); err != nil {
return PathGroup{}, err
}
g, ok := groups[path]
if !ok {
return PathGroup{}, errors.New("no group in this test for " + path)
}
return g, nil
}
t.Cleanup(func() { GroupOfPath = was })
rootGroupsAre(t, map[int]string{0: "root", 10: "wheel", 968: "docker"})
}
// rootGroupsAre stands in for the numbers this machine gives the groups whose members act as root.
func rootGroupsAre(t *testing.T, ids map[int]string) {
t.Helper()
was := RootGroupIDs
RootGroupIDs = func() map[int]string { return ids }
t.Cleanup(func() { RootGroupIDs = was })
}
func accessGranting(id, path, account string, containers ...string) string {
grant := `"grant":{"account":"` + account + `"`
if len(containers) > 0 {
grant += `,"containers":["` + strings.Join(containers, `","`) + `"]`
}
return `{"id":"` + id + `","type":"access","path":"` + path + `","mode":"read-write",` + grant + `}}`
}
func mounting(id, path string) string {
return `{"id":"` + id + `","type":"container","name":"` + strings.ReplaceAll(id, ".", "-") + `","image":"` + pinned +
`","volumes":["` + path + `:/library"]}`
}
func applyGrants(t *testing.T, m *grantMachine, known store.State, resources ...string) (Report, store.State, error) {
t.Helper()
return Apply(context.Background(), archHost(t), parse(t, `{"declaration":1,"resources":[`+
strings.Join(resources, ",")+`]}`), known, store.OriginDeclared, m.run, nil, nil)
}
func statOf(t *testing.T, path string) string {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
uid, gid, _ := OwnerOf(info)
return fmt.Sprintf("%s %d:%d %s", info.Mode(), uid, gid, info.ModTime())
}
func TestAReadWriteAccessGrantsItsPathsGroupToTheContainerAndTheAccountAndLeavesThePath(t *testing.T) {
library := t.TempDir()
if err := os.WriteFile(filepath.Join(library, "album"), []byte("x"), 0o640); err != nil {
t.Fatal(err)
}
groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
before, beforeInside := statOf(t, library), statOf(t, filepath.Join(library, "album"))
m := newGrantMachine("operator", []string{"operator"}, "media")
report, state, err := applyGrants(t, m, store.State{},
accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(" "+m.groups()+" ", " media ") {
t.Fatalf("the account was not put in the path's group: %q", m.groups())
}
if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") {
t.Fatalf("the container was not created with the path's group: %s", run)
}
o := outcomeOf(report, "importer.music")
if o.Action != "updated" || !strings.Contains(o.Detail, "grants media (gid 1001)") ||
!strings.Contains(o.Detail, "put operator in media") {
t.Errorf("the access did not say what it granted: %+v", o)
}
if rec, _ := state.Find("importer.music"); rec.User != "operator" || strings.Join(rec.Groups, " ") != "media" {
t.Errorf("the access did not record the group it put the account in: %+v", rec)
}
// The path, and what is under it, exactly as they were (ADR 0312 §2).
if after := statOf(t, library); after != before {
t.Errorf("the accessed path changed: %s, was %s", after, before)
}
if after := statOf(t, filepath.Join(library, "album")); after != beforeInside {
t.Errorf("a file under the accessed path changed: %s, was %s", after, beforeInside)
}
for _, asked := range m.asked {
for _, verb := range []string{"chown", "chmod", "chgrp", "setfacl"} {
if strings.HasPrefix(asked, verb) || strings.Contains(asked, " "+verb+" ") {
t.Errorf("the engine asked the machine to change something: %q", asked)
}
}
}
// Applied again: nothing moves, and nothing is recreated.
m.runs = map[string][]string{}
report, _, err = applyGrants(t, m, state,
accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library))
if err != nil {
t.Fatal(err)
}
if o := outcomeOf(report, "importer.music"); o.Action != "unchanged" {
t.Errorf("a second apply moved the access: %+v", o)
}
if len(m.runs) != 0 {
t.Errorf("a second apply recreated the container: %v", m.runs)
}
}
func TestAReadAccessGrantsNothing(t *testing.T) {
library := t.TempDir()
groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
// A grant on a read access is refused whole: the controller asked for something the decision forbids.
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"player.music","type":"access","path":"` +
library + `","mode":"read","grant":{"account":"operator"}}]}`))
if err == nil || !strings.Contains(err.Error(), "a read access grants nothing") {
t.Fatalf("a grant on a read access was not refused by name: %v", err)
}
m := newGrantMachine("operator", []string{"operator"}, "media")
report, state, err := applyGrants(t, m, store.State{},
`{"id":"player.music","type":"access","path":"`+library+`","mode":"read"}`, mounting("player.app", library))
if err != nil {
t.Fatal(err)
}
if strings.Contains(m.groups(), "media") {
t.Errorf("a read access put the account in a group: %q", m.groups())
}
if run := strings.Join(m.runs["player-app"], " "); strings.Contains(run, "--group-add") {
t.Errorf("a read access gave its container a group: %s", run)
}
if o := outcomeOf(report, "player.music"); o.Action != "unchanged" {
t.Errorf("a read access moved something: %+v", o)
}
if rec, _ := state.Find("player.music"); len(rec.Groups) != 0 || rec.User != "" {
t.Errorf("a read access recorded a grant: %+v", rec)
}
}
func TestAnAccessWhoseGroupCannotWriteGrantsNothingAndSaysTheOperatorDecides(t *testing.T) {
spool := t.TempDir()
groupOfPaths(t, map[string]PathGroup{spool: {GID: 1002, Name: "downloads", Writable: false}})
before := statOf(t, spool)
m := newGrantMachine("operator", nil, "downloads")
report, _, err := applyGrants(t, m, store.State{},
accessGranting("nzbget.downloads", spool, "operator", "nzbget.app"), mounting("nzbget.app", spool))
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "nzbget.downloads")
if !strings.Contains(o.Detail, "cannot write it, so nothing is granted") || !strings.Contains(o.Detail, "the operator decides") ||
!strings.Contains(o.Detail, "nothing was changed") {
t.Errorf("the access did not say its group cannot write and that the operator decides: %+v", o)
}
// Nothing is given (ADR 0312 §2): no account group, no container group.
if strings.Contains(m.groups(), "downloads") {
t.Errorf("a group that cannot write was granted to the account: %q", m.groups())
}
if run := strings.Join(m.runs["nzbget-app"], " "); strings.Contains(run, "--group-add") {
t.Errorf("a group that cannot write was granted to the container: %s", run)
}
if after := statOf(t, spool); after != before {
t.Errorf("the engine changed a path whose group cannot write: %s, was %s", after, before)
}
// The words the look states it unhealthy with: the remedy is the operator's, and the path is never in them.
words := CannotWriteWords(PathGroup{GID: 1002, Name: "downloads"})
if !strings.Contains(words, "the operator's to decide") || strings.Contains(words, spool) {
t.Errorf("the unhealthy reason is %q", words)
}
}
func TestTheRootGroupIsNeverGranted(t *testing.T) {
for _, g := range []PathGroup{{GID: 0, Name: "root", Writable: true}, {GID: 968, Name: "docker", Writable: true},
{GID: 10, Name: "wheel", Writable: true}, {GID: 42, Name: "shadow", Writable: true},
// A number the root groups have here under another name — judged by the number, never the name.
{GID: 968, Name: "containers", Writable: true},
// A number the machine names nothing for: unjudged, so never granted, to a container either.
{GID: 4242, Name: "", Writable: true}} {
want := g.Name
if want == "" || want == "containers" {
want = fmt.Sprintf("gid %d", g.GID)
}
t.Run(fmt.Sprintf("%d %s", g.GID, g.Name), func(t *testing.T) {
shared := t.TempDir()
groupOfPaths(t, map[string]PathGroup{shared: g})
m := newGrantMachine("operator", nil, g.Name)
report, _, err := applyGrants(t, m, store.State{},
accessGranting("importer.shared", shared, "operator", "importer.app"), mounting("importer.app", shared))
if err == nil {
t.Fatal("a read-write access to a path of a root-granting group applied")
}
var failed *Error
if !errors.As(err, &failed) || failed.Resource != "importer.shared" ||
!strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never") {
t.Errorf("the refusal does not name the access and the group: %v", err)
}
if g.Name != "" && strings.Contains(m.groups(), g.Name) {
t.Errorf("the account was put in %s", g.Name)
}
if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") {
t.Errorf("the container was given a root-granting group: %s", run)
}
_ = report
})
}
}
func TestAGrantedGroupIsTakenBackUnlessStillAskedForOrHeldBefore(t *testing.T) {
music, movies := t.TempDir(), t.TempDir()
groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true},
movies: {GID: 1001, Name: "media", Writable: true}})
t.Run("no longer asked", func(t *testing.T) {
m := newGrantMachine("operator", []string{"operator"}, "media")
_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
if err != nil {
t.Fatal(err)
}
report, state, err := applyGrants(t, m, state, mounting("other.app", t.TempDir()))
if err != nil {
t.Fatal(err)
}
if strings.Contains(m.groups(), "media") {
t.Errorf("the group the mesh granted was not taken back: %q", m.groups())
}
if o := outcomeOf(report, "importer.music"); o.Action != "restored" || !strings.Contains(o.Detail, "taken out of media") {
t.Errorf("the removal did not say it took the group back: %+v", o)
}
if _, still := state.Find("importer.music"); still {
t.Error("the access is still recorded")
}
})
t.Run("still asked by another access", func(t *testing.T) {
m := newGrantMachine("operator", []string{"operator"}, "media")
_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"),
accessGranting("sonarr.movies", movies, "operator"))
if err != nil {
t.Fatal(err)
}
report, _, err := applyGrants(t, m, state, accessGranting("sonarr.movies", movies, "operator"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(m.groups(), "media") {
t.Errorf("a group another access still grants was taken back: %q", m.groups())
}
if o := outcomeOf(report, "importer.music"); !strings.Contains(o.Detail, "still asks for") {
t.Errorf("the removal did not say why the group stayed: %+v", o)
}
})
t.Run("still asked by a user resource", func(t *testing.T) {
m := newGrantMachine("operator", []string{"operator"}, "media")
_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
if err != nil {
t.Fatal(err)
}
if _, _, err := applyGrants(t, m, state,
`{"id":"player.account","type":"user","name":"operator","groups":["media"]}`); err != nil {
t.Fatal(err)
}
if !strings.Contains(m.groups(), "media") {
t.Errorf("a group a user resource asks for was taken back: %q", m.groups())
}
})
t.Run("held before", func(t *testing.T) {
m := newGrantMachine("operator", []string{"operator", "media"})
_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
if err != nil {
t.Fatal(err)
}
if rec, _ := state.Find("importer.music"); len(rec.Groups) != 0 {
t.Fatalf("a group the account held before was recorded as the mesh's: %+v", rec)
}
if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil {
t.Fatal(err)
}
if !strings.Contains(m.groups(), "media") {
t.Errorf("a group the account held before the mesh was taken back: %q", m.groups())
}
})
}
func TestAContainerIsRecreatedWithTheGroupItIsGranted(t *testing.T) {
library := t.TempDir()
groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
m := newGrantMachine("operator", nil, "media")
// Running before its access granted anything: a controller older than the grant.
_, state, err := applyGrants(t, m, store.State{},
`{"id":"importer.music","type":"access","path":"`+library+`","mode":"read-write"}`, mounting("importer.app", library))
if err != nil {
t.Fatal(err)
}
if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") {
t.Fatalf("a container granted nothing was given a group: %s", run)
}
report, _, err := applyGrants(t, m, state,
accessGranting("importer.music", library, "", "importer.app"), mounting("importer.app", library))
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "importer.app")
if o.Action != "updated" || !strings.Contains(o.Detail, "recreated to hold the groups") {
t.Errorf("the container was not recreated for its group: %+v", o)
}
if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") {
t.Errorf("the recreated container does not hold the group: %s", run)
}
}
func TestTheToolRunnerIsRestartedOnceItsAccountHoldsTheGroup(t *testing.T) {
me, err := user.Current()
if err != nil {
t.Skip("no current account to run the tool runner as")
}
units, bundles := t.TempDir(), t.TempDir()
wasUnits, wasBundles := unitDir, daemonRoot
unitDir, daemonRoot = units, bundles
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
library := t.TempDir()
groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
m := newGrantMachine(me.Username, nil, "media")
body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
runner := `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
`","digest":"` + digest + `","run":["node","main.js"],"user":"` + me.Username +
`","restart-on":["importer.music"]}`
// Running before the grant.
_, state, err := applyGrants(t, m, store.State{}, runner)
if err != nil {
t.Fatal(err)
}
m.restarts = nil
report, state, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner)
if err != nil {
t.Fatal(err)
}
if len(m.restarts) != 1 || !strings.HasPrefix(m.restarts[0], "node-tools.service: ") ||
!strings.Contains(m.restarts[0], "media") {
t.Fatalf("the tool runner was not restarted once its account held the group: %v", m.restarts)
}
if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "importer.music") {
t.Errorf("the restart does not say what it was for: %+v", o)
}
// And not again while nothing moves.
m.restarts = nil
if _, _, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner); err != nil {
t.Fatal(err)
}
if len(m.restarts) != 0 {
t.Errorf("the tool runner was restarted with nothing moved: %v", m.restarts)
}
}
// sessionReader is an account whose own service manager runs, holding the groups it began with.
type sessionReader struct{ database, session []string }
func (s sessionReader) InDatabase(context.Context, string) ([]string, error) { return s.database, nil }
func (s sessionReader) Session(_ context.Context, _ string, groups []string) (accounts.Session, error) {
has := map[string]bool{}
for _, g := range s.session {
has[g] = true
}
return accounts.Session{Running: true, Has: has}, nil
}
func (s sessionReader) Escalation(context.Context, string, []string) ([]string, error) {
return nil, nil
}
func TestAnAccountsOwnSessionIsSaidReloginNeededForAGrantedGroup(t *testing.T) {
library := t.TempDir()
groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
d := parse(t, `{"declaration":1,"resources":[`+accessGranting("importer.music", library, "operator")+`]}`)
as := GrantedAccounts(d, nil)
if len(as) != 1 || as[0].Module != "importer" || as[0].Name != "operator" || strings.Join(as[0].Groups, " ") != "media" {
t.Fatalf("the account the access grants is not judged: %+v", as)
}
j := accounts.New(sessionReader{database: []string{"operator", "media"}, session: []string{"operator"}})
j.Set(as)
st, _ := j.Look(context.Background())
if len(st.Accounts) != 1 || !strings.HasPrefix(st.Accounts[0].Reason, accounts.ReasonRelogin) {
t.Errorf("a session of the account's own manager that began before the grant is not said relogin needed: %+v",
st.Accounts)
}
// A read access, or one refused, asks nothing of the judge.
groupOfPaths(t, map[string]PathGroup{library: {GID: 0, Name: "root", Writable: true}})
if as := GrantedAccounts(d, nil); len(as) != 0 {
t.Errorf("a refused grant is judged: %+v", as)
}
}
// An access going while a user resource still asks for its group hands the group over: the user resource records it
// as the mesh's, and takes it back when it stops asking too (novox/hq ADR 0312 §5, ADR 0252 §3).
func TestAGroupStillAskedForIsHandedOverAndTakenBackWhenTheLastAskerGoes(t *testing.T) {
music := t.TempDir()
groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}})
player := `{"id":"player.account","type":"user","name":"operator","groups":["media"]}`
m := newGrantMachine("operator", []string{"operator"}, "media")
_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), player)
if err != nil {
t.Fatal(err)
}
if rec, _ := state.Find("player.account"); len(rec.Groups) != 0 {
t.Fatalf("the user resource recorded a group the access put the account in: %+v", rec)
}
_, state, err = applyGrants(t, m, state, player)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(m.groups(), "media") {
t.Fatalf("a group the user resource still asks for was taken back: %q", m.groups())
}
if rec, _ := state.Find("player.account"); strings.Join(rec.Groups, " ") != "media" {
t.Fatalf("the group was not handed to the user resource that still asks for it: %+v", rec)
}
if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil {
t.Fatal(err)
}
if strings.Contains(m.groups(), "media") {
t.Errorf("the group stayed after the last resource asking for it went: %q", m.groups())
}
}
// A grant's account is an argument to id, usermod and gpasswd, and a name that starts with a dash would be read as
// an option by each.
func TestAGrantsAccountMayNotReadAsAnOption(t *testing.T) {
for _, name := range []string{"-ogroups", "--help", "a b", "a:b"} {
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` +
`"mode":"read-write","grant":{"account":"` + name + `"}}]}`))
if err == nil || !strings.Contains(err.Error(), "not starting with a dash") {
t.Errorf("the account %q was accepted: %v", name, err)
}
}
if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` +
`"mode":"read-write","grant":{"account":"jochen"}}]}`)); err != nil {
t.Errorf("an ordinary account was refused: %v", err)
}
}
// The scheduler reads no path issue 339's guard refuses, as the apply reads none.
func TestAScheduledStepIsArmedWithoutReadingARefusedPath(t *testing.T) {
var read []string
was := GroupOfPath
GroupOfPath = func(path string) (PathGroup, error) {
read = append(read, path)
return PathGroup{GID: 1001, Name: "media", Writable: true}, nil
}
t.Cleanup(func() { GroupOfPath = was })
rootGroupsAre(t, map[int]string{0: "root"})
d := parse(t, `{"declaration":1,"resources":[`+accessGranting("m.etc", "/etc", "", "m.step")+`,`+
`{"id":"m.step","type":"container","name":"step","image":"`+pinned+`","schedule":"0 3 * * *","volumes":["/etc:/x"]}]}`)
if got := ContainerGroupsOf(d); len(got["m.step"]) != 0 {
t.Errorf("a step was armed with the group of a refused path: %v", got)
}
if len(read) != 0 {
t.Errorf("the scheduler read a path the guard refuses: %v", read)
}
}
// The group database is read as name to number, so a number carrying a root group's name under any other name
// is still that number.
func TestGroupEntriesAreReadByNumber(t *testing.T) {
got := groupEntries("root:x:0:\n# a comment\ndocker:x:968:operator\ncontainers:x:968:\nbroken\n")
if got["root"] != 0 || got["docker"] != 968 || got["containers"] != 968 || len(got) != 3 {
t.Errorf("read %v", got)
}
}
internal/apply/identities.go
+253
View File
@@ -0,0 +1,253 @@
package apply
import (
"context"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// Every module is given an identity of its own (novox/hq ADR 0325 §3, §11, issue 567).
//
// The controller allocates each module a user and a group number once, from a range of its own, the same on every
// machine, and declares them as `mesh-<module>`. The engine makes the group and the account with exactly those
// numbers — a system account: no login shell, no home, no password — before anything of the declaration is
// applied, so a directory the module owns can be given to it and a container told to run as it.
//
// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is
// refused for the whole declaration before anything is made or started, naming why — for an account and for an
// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is
// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is.
//
// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the
// engine made the account, the group or both, and an account or a group that was there before is never removed.
// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the
// runner, which is what a test stands in for.
type accountsTool interface {
CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int) error
CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error
RemoveAccount(ctx context.Context, run system.Runner, name string) error
RemoveGroup(ctx context.Context, run system.Runner, name string) error
}
// entry is one line the user or group database gives for a key: its name and its numbers.
type entry struct {
name string
id, prim int
}
// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a
// database that does not answer is an error, never read as absent (system.LookUpUser's rule).
func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) {
out, err := run(ctx, "getent", database, key)
if err != nil {
if code, ok := system.ExitCode(err); ok && code == 2 {
return entry{}, false, nil
}
return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err)
}
fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":")
if len(fields) < 3 {
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
}
e := entry{name: fields[0], prim: -1}
if e.id, err = strconv.Atoi(fields[2]); err != nil {
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
}
if database == "passwd" && len(fields) >= 4 {
if e.prim, err = strconv.Atoi(fields[3]); err != nil {
e.prim = -1
}
}
return e, true, nil
}
// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before
// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started.
func refuseRootIdentities(d *declaration.Declaration) error {
var problems []string
for _, id := range d.Identities {
if err := rootIdentity(id.UID, id.GID); err != nil {
problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err))
}
}
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && c.RunAs != nil {
if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil {
problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v",
c.ID, c.RunAs.UID, c.RunAs.GID, err))
}
}
}
if len(problems) == 0 {
return nil
}
sort.Strings(problems)
return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; "))
}
// applyIdentity makes one identity as declared, or refuses it naming whose the name or the numbers are. What it
// made is recorded on known; an identity already right is unchanged.
func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, known *store.State,
run Runner) (Outcome, error) {
out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"}
user, hasUser, err := lookUp(ctx, run, "passwd", id.Name)
if err != nil {
return out, err
}
group, hasGroup, err := lookUp(ctx, run, "group", id.Name)
if err != nil {
return out, err
}
byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID))
if err != nil {
return out, err
}
byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.GID))
if err != nil {
return out, err
}
const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0325 §3)"
switch {
case hasUser && (user.id != id.UID || user.prim != id.GID):
return out, fmt.Errorf("the account %s is here with uid %d and group %d, and the mesh allocated uid %d, "+
"gid %d%s", id.Name, user.id, user.prim, id.UID, id.GID, never)
case hasGroup && group.id != id.GID:
return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d%s", id.Name,
group.id, id.GID, never)
case uidUsed && byUID.name != id.Name:
return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID,
byUID.name, id.Name, never)
case gidUsed && byGID.name != id.Name:
return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.GID,
byGID.name, id.Name, never)
}
rec := madeIdentity(known, id.Name)
rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID
var made []string
if !hasGroup {
if err := tool.CreateSystemGroup(ctx, run, id.Name, id.GID); err != nil {
return out, err
}
rec.MadeGroup = true
made = append(made, fmt.Sprintf("the group (gid %d)", id.GID))
}
if !hasUser {
if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID); err != nil {
recordIdentity(known, rec)
return out, err
}
rec.MadeUser = true
made = append(made, fmt.Sprintf("the account (uid %d), with no login, no home and no password", id.UID))
}
if rec.At.IsZero() {
rec.At = time.Now().UTC()
}
recordIdentity(known, rec)
if len(made) > 0 {
out.Action, out.Detail = "created", "made "+strings.Join(made, " and ")
} else {
out.Detail = "here with its numbers"
}
return out, nil
}
// madeIdentity is what the engine recorded of name, or an empty record.
func madeIdentity(known *store.State, name string) store.MadeIdentity {
for _, m := range known.Identities {
if m.Name == name {
return m
}
}
return store.MadeIdentity{}
}
// recordIdentity records what was made of one identity, in place of any record before.
func recordIdentity(known *store.State, rec store.MadeIdentity) {
for i, m := range known.Identities {
if m.Name == rec.Name {
known.Identities[i] = rec
return
}
}
if rec.MadeUser || rec.MadeGroup {
known.Identities = append(known.Identities, rec)
}
}
// removeIdentities removes what the engine made of every identity no longer declared — the account, then the
// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed
// stays recorded, said, for the next apply.
func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State,
run Runner) []Outcome {
wanted := map[string]bool{}
for _, id := range declared {
wanted[id.Name] = true
}
var out []Outcome
var keep []store.MadeIdentity
for _, m := range known.Identities {
if wanted[m.Name] {
keep = append(keep, m)
continue
}
o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"}
var said []string
if m.MadeUser {
if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) {
said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+
"database did not answer")
m.MadeUser = false
} else if has {
if err := tool.RemoveAccount(ctx, run, m.Name); err != nil {
said = append(said, "the account could not be removed: "+err.Error())
} else {
m.MadeUser = false
said = append(said, "removed the account")
}
} else {
m.MadeUser = false
}
}
if m.MadeGroup && !m.MadeUser {
if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) {
said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+
"database did not answer")
m.MadeGroup = false
} else if has {
if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
said = append(said, "the group could not be removed: "+err.Error())
} else {
m.MadeGroup = false
said = append(said, "removed the group")
}
} else {
m.MadeGroup = false
}
}
if m.MadeUser || m.MadeGroup {
o.Action = "waiting"
keep = append(keep, m)
}
o.Detail = "no longer declared: " + strings.Join(said, "; ")
out = append(out, o)
}
known.Identities = keep
return out
}
// toolFor is the accounts tool of a system, or why it has none.
func toolFor(sys system.System) (accountsTool, error) {
if t, ok := sys.(accountsTool); ok {
return t, nil
}
return nil, errors.New("this system makes no module identity: it has no way to make a system account")
}
internal/apply/identities_test.go
+287
View File
@@ -0,0 +1,287 @@
package apply
import (
"context"
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with
// the numbers the controller allocated, a container told to run as it, and never a root number.
// accountsMachine is a user and group database the runner answers from, and every command that changed it.
type accountsMachine struct {
users, groups map[string][2]int // name → id, primary gid (a group's second is unused)
ran []string
}
func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
switch name {
case "getent":
db, key := args[0], args[1]
table := m.users
if db == "group" {
table = m.groups
}
for n, ids := range table {
if n == key || strconv.Itoa(ids[0]) == key {
if db == "group" {
return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil
}
return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::/:/usr/bin/nologin\n", nil
}
}
return "", errors.New("getent exited 2")
case "groupadd":
m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0}
case "useradd":
m.users[args[len(args)-1]] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])}
case "userdel":
delete(m.users, args[0])
case "groupdel":
delete(m.groups, args[0])
default:
return "", nil
}
m.ran = append(m.ran, line)
return "", nil
}
func mustAtoi(s string) int {
n, _ := strconv.Atoi(s)
return n
}
func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration {
t.Helper()
d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` +
resources + `]}`))
if err != nil {
t.Fatal(err)
}
return d
}
// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was
// made — when no longer declared.
func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) {
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
dir := t.TempDir()
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`)
report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
if err != nil {
t.Fatal(err)
}
want := []string{
"groupadd --system --gid 61001 mesh-radarr",
"useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir / --shell /usr/bin/nologin mesh-radarr",
}
if strings.Join(m.ran, "\n") != strings.Join(want, "\n") {
t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n"))
}
if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" {
t.Errorf("the identity's outcome: %+v", o)
}
if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup {
t.Fatalf("what was made is not recorded: %+v", known.Identities)
}
// Applied again: nothing is made twice.
m.ran = nil
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil {
t.Fatal(err)
}
if len(m.ran) != 0 {
t.Fatalf("an identity already here was made again: %v", m.ran)
}
// Undeclared, with a group the machine had before for another identity: only what was made goes.
m.groups["mesh-sonarr"] = [2]int{61002, 0}
other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`,
`{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`)
if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil {
t.Fatal(err)
}
if _, has := m.users["mesh-radarr"]; has {
t.Error("an identity no longer declared kept its account")
}
if _, has := m.groups["mesh-radarr"]; has {
t.Error("an identity no longer declared kept its group")
}
if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup {
t.Error("a group that was there before is recorded as the mesh's")
}
}
// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the
// module is not attempted; the rest of the machine is.
func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) {
dir := t.TempDir()
for _, tc := range []struct {
users, groups map[string][2]int
says string
}{
{users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"},
{users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"},
{users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"},
} {
m := &accountsMachine{users: tc.users, groups: tc.groups}
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
`{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+
`{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`)
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
if err == nil || !strings.Contains(err.Error(), tc.says) {
t.Errorf("not refused naming %s: %v", tc.says, err)
}
if len(m.ran) != 0 {
t.Errorf("something was made for an identity that is another's: %v", m.ran)
}
if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" {
t.Errorf("the module of a refused identity was attempted: %+v", o)
}
if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil {
t.Errorf("the rest of the machine was not applied: %v", err)
}
os.RemoveAll(filepath.Join(dir, "plex"))
}
}
// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is
// made or started. A number 0 is refused on arrival, by the parser.
func TestNoModuleIdentityIsRoot(t *testing.T) {
was := RootGroupIDs
RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} }
defer func() { RootGroupIDs = was }()
dir := t.TempDir()
for _, d := range []*declaration.Declaration{
withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`,
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+
`{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+
`"identity":{"mode":"run","uid":61001,"gid":10}}`),
} {
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") {
t.Errorf("a root group was not refused naming why: %v", err)
}
if len(m.ran) != 0 {
t.Errorf("something was made or started: %v", m.ran)
}
if _, err := os.Stat(filepath.Join(dir, "config")); err == nil {
t.Error("a resource of a declaration naming a root group was applied")
}
}
for _, raw := range []string{
`{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`,
`{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` +
`"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`,
} {
if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") {
t.Errorf("a number 0 was not refused on arrival: %v", err)
}
}
}
// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user
// option — and a changed identity is a container recreated.
func TestAContainerIsToldItsModulesIdentity(t *testing.T) {
env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{
Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}}
run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{
Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}}
if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") {
t.Errorf("an image told by environment was not: %s", got)
}
if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") {
t.Errorf("an image told by the runtime's user option was not: %s", got)
}
plain := *env
plain.RunAs = nil
moved := *env
moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID",
UID: 61009, GID: 61009}
if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) ||
containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) {
t.Error("an identity told, or renumbered, does not recreate the container")
}
}
// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of
// the access, and nothing is granted again.
func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) {
dir := t.TempDir()
groups := "nox media"
var ran []string
run := func(_ context.Context, name string, args ...string) (string, error) {
ran = append(ran, name+" "+strings.Join(args, " "))
switch name {
case "id":
return groups + "\n", nil
case "gpasswd":
groups = "nox"
}
return "", nil
}
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`)
report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") {
t.Fatalf("the grant's group was not taken back: %v", ran)
}
if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") {
t.Errorf("the take-back is not said: %+v", o)
}
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 {
t.Errorf("the record still holds the grant's groups: %+v", rec)
}
}
// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again.
func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) {
dir := t.TempDir()
refuse := true
groups := "nox media"
run := func(_ context.Context, name string, args ...string) (string, error) {
switch name {
case "id":
return groups + "\n", nil
case "gpasswd":
if refuse {
return "", errors.New("gpasswd exited 1: locked")
}
groups = "nox"
}
return "", nil
}
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`)
_, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 {
t.Fatalf("a group that could not be left was forgotten: %+v", rec)
}
refuse = false
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil {
t.Fatal(err)
}
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" {
t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups)
}
}
internal/apply/keep.go
+953
View File
@@ -0,0 +1,953 @@
package apply
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// A shared folder is kept usable through access entries (novox/hq ADR 0325, issue 567).
//
// Several modules use one folder of the operator's — a library one imports into, one renames in, one plays from —
// each as its own identity. The engine keeps, for every placed folder, one ACL entry per identity that uses it,
// on the folder and as its default for everything made under it, and changes nothing else of the path (§5).
//
// **When it may keep a folder.** Only one that is the mesh's to keep: taken on the operator's warrant (§8), or
// holding no data yet — empty when first seen, or made by the engine (§9). A folder that holds data and was never
// taken is stated waiting, with a wait of kind access, until the operator lets the mesh keep it (§7); one whose disk
// is not there waits for the disk (§9); root's data on its placed filesystem is never kept, and is unhealthy (§12).
//
// **What every apply does.** It judges the folder (assessFolder), makes a missing one where its data lives, and
// sets the users' entries on the folder itself. What lies under a taken folder is the walks' (ReconcileKept): a
// user added has its entries added to everything under it, a user dropped has exactly its own removed, outside
// the apply and bounded. The watch and the sweep keep what is made later usable (§6, §10).
// keptRecords is where the engine keeps what it knows of each kept or seen folder: the file beside the state in
// the engine, memory in a test.
type keptRecords interface {
All() (map[string]store.KeptFolder, error)
Update(path string, change func(f *store.KeptFolder, had bool) bool) error
}
// KeptFolders is the engine's record of the folders it keeps. The engine sets the file beside its state.
var KeptFolders keptRecords = NewKeptMemory()
// keptMemory is a keptRecords held in memory.
type keptMemory struct {
mu sync.Mutex
m map[string]store.KeptFolder
}
// NewKeptMemory is an empty record held in memory: a test's, or a one-shot apply's.
func NewKeptMemory() keptRecords { return &keptMemory{m: map[string]store.KeptFolder{}} }
func (k *keptMemory) All() (map[string]store.KeptFolder, error) {
k.mu.Lock()
defer k.mu.Unlock()
out := map[string]store.KeptFolder{}
for p, f := range k.m {
out[p] = f
}
return out, nil
}
func (k *keptMemory) Update(path string, change func(f *store.KeptFolder, had bool) bool) error {
k.mu.Lock()
defer k.mu.Unlock()
f, had := k.m[path]
f.Path = path
if change(&f, had) {
k.m[path] = f
} else {
delete(k.m, path)
}
return nil
}
// keptRecord is the record of path, and whether there is one. A record that cannot be read is none: the folder
// is judged afresh, which waits rather than keeps.
func keptRecord(path string) (store.KeptFolder, bool) {
all, err := KeptFolders.All()
if err != nil {
return store.KeptFolder{}, false
}
f, had := all[path]
return f, had
}
// folderWant is what every access naming one folder asks of it: the accesses, each uid at the widest use any of
// them gives it (ADR 0325 §4: the tool runner's account stands for every module whose tools use it), and whether
// the controller says it was taken.
type folderWant struct {
path string
ids []string
users map[uint32]store.KeptUser
taken bool
}
// foldersOf is every declared access's folder, by its clean path. A folder the guard refuses is not read (issue
// 339).
func foldersOf(resources []declaration.Resource, refused map[string]error) map[string]*folderWant {
out := map[string]*folderWant{}
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok {
continue
}
path := filepath.Clean(a.Path)
if refused[path] != nil {
continue
}
f := out[path]
if f == nil {
f = &folderWant{path: path, users: map[uint32]store.KeptUser{}}
out[path] = f
}
f.ids = append(f.ids, a.ID)
f.taken = f.taken || a.Taken
for _, u := range a.Users {
had, seen := f.users[uint32(u.UID)]
if seen && (had.Use == declaration.AccessReadWrite || u.Use != declaration.AccessReadWrite) {
continue
}
f.users[uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID), Use: u.Use, From: u.From}
}
}
return out
}
// list is the users, by uid.
func (f *folderWant) list() []store.KeptUser {
if f == nil {
return nil
}
out := make([]store.KeptUser, 0, len(f.users))
for _, u := range f.users {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].UID < out[j].UID })
return out
}
// usersOf is a list of users as each uid and whether it writes.
func usersOf(us []store.KeptUser) map[uint32]bool {
out := map[uint32]bool{}
for _, u := range us {
out[u.UID] = out[u.UID] || u.Use == declaration.AccessReadWrite
}
return out
}
// sameUsers is whether two lists give the same uids the same use.
func sameUsers(a, b []store.KeptUser) bool {
x, y := usersOf(a), usersOf(b)
if len(x) != len(y) {
return false
}
for uid, rw := range x {
if w, ok := y[uid]; !ok || w != rw {
return false
}
}
return true
}
// accessName is the module and its own id for an access, from the resource id `<module>.<id>`.
func accessName(id string) (module, name string) {
if m, ok := moduleOf(id); ok {
return m, id[len(m)+1:]
}
return "", id
}
// KeepWords and DiskWords are what an access wait says the operator gives (ADR 0325 §7): never a path.
func KeepWords(name string) string { return "let the mesh keep the shared folder " + name }
func DiskWords(name string) string { return "mount the disk that holds " + name }
// The judgements of a folder.
const (
// folderKeep is a folder the mesh keeps: taken, or holding no data yet.
folderKeep = "keep"
// folderMake is a folder absent under a parent on its placed filesystem, not root's: made, then kept.
folderMake = "make"
// folderWaitKeep is a folder that holds data and was never taken: it waits for the operator's word.
folderWaitKeep = "wait-keep"
// folderWaitDisk is a folder whose disk is not there.
folderWaitDisk = "wait-disk"
// folderRoot is root's data on its placed filesystem: never kept.
folderRoot = "root"
// folderUnknown is a folder whose filesystem could not be named: left as it is until it is.
folderUnknown = "unknown"
)
// folderVerdict is the judgement of one folder, with what it read.
type folderVerdict struct {
kind string
// how is why a folder kept is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptMade.
how string
// mount is the mount point that holds what is there.
mount string
// why is a root verdict's or a broken folder's reason; err a folder that is not one (a file, a link).
why string
err error
// moved is that the folder's recorded mount is mounted and another holds it: it is judged afresh.
moved bool
// unnamed is a folder whose filesystem cannot be named: never kept, nor taken.
unnamed bool
}
// assessFolder judges a folder as an apply, a look and every walk do (ADR 0325 §7, §9, §12): its disk first, then
// whose it is, then whether the mesh may keep it. Its words carry no path: they reach the operator's channel.
func assessFolder(ctx context.Context, path string, rec store.KeptFolder, taken bool) folderVerdict {
others := recordedMounts(path)
info, err := lstatWithin(path)
if errors.Is(err, os.ErrNotExist) {
m, err := NearestAncestor(path)
if err != nil {
return folderVerdict{err: err}
}
if !m.Info.IsDir() {
return folderVerdict{err: errors.New("the folder is not there, and its nearest existing parent is not " +
"a folder (a file or a link); nothing is made under it")}
}
pl := placedOn(ctx, path, m.Ancestor, rec.Mount, others)
if pl.Away {
return folderVerdict{kind: folderWaitDisk, mount: pl.Mount}
}
uid, gid, ok := statOwner(m.Info)
if !ok {
return folderVerdict{err: errors.New("the owner of the folder's nearest existing parent cannot be read " +
"on this machine")}
}
if why := rootOwned(uint32(uid), uint32(gid)); why != "" {
return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is not there, and its nearest " +
"existing parent is " + why}
}
return folderVerdict{kind: folderMake, mount: pl.Mount, how: store.KeptMade, moved: pl.Moved}
}
if err != nil {
return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))}
}
if !info.IsDir() {
return folderVerdict{err: errors.New("the folder is not a directory, and an access is a shared directory")}
}
pl := placedOn(ctx, path, path, rec.Mount, others)
if pl.Away {
return folderVerdict{kind: folderWaitDisk, mount: pl.Mount}
}
// Another filesystem at the same mount point (another pool, another disk) is a move too. A record from an earlier
// build names its filesystem another way, and is not compared: the next keep names it this way.
here := PlacedID(pl.Mount)
// **A filesystem that cannot be named now is unknown, never a move** (a name read that did not answer, a
// filesystem with no UUID, id or source): the folder holds what it is — nothing walked, nothing newly kept,
// nothing reset — and is judged again when it is named. One never kept is so never kept: it fails closed.
if here == "" {
return folderVerdict{kind: folderUnknown, mount: pl.Mount, unnamed: true,
why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"}
}
moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here)
facts, err := readFolder(path)
if err != nil {
return folderVerdict{err: errors.New("the folder cannot be read: " + pathless(err))}
}
if why := rootOwned(facts.UID, facts.GID); why != "" {
return folderVerdict{kind: folderRoot, mount: pl.Mount, why: "it is " + why}
}
// The controller's word that the folder was taken holds for the filesystem the warrant was carried out on: a
// folder moved to another is not taken, and asks again (§8).
takenHere := taken && (rec.TakenOn == here || (rec.TakenOn == "" && !moved))
switch {
case rec.Kept && !moved:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: rec.How}
case takenHere:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptByWarrant, moved: moved}
case facts.Empty:
return folderVerdict{kind: folderKeep, mount: pl.Mount, how: store.KeptEmpty, moved: moved}
}
// What is there now holds data the mesh never took — a disk mounted over a folder kept while it was empty
// below it included (a move): it waits for the operator's word.
return folderVerdict{kind: folderWaitKeep, mount: pl.Mount, moved: moved}
}
// movedRecord is f judged afresh on the filesystem now at mount: the warrant's filesystem is kept so the
// controller's word is not read for the new one, and nothing else of what was kept there holds. What the mesh
// added on the old filesystem is left to be taken out once it holds the folder again (Left) only where it was
// walked onto data under a warrant: a folder kept because it was empty, or made, carries entries on that empty
// folder alone, hidden under the mount point and granting nothing on data, and they are forgotten (said once, by
// the apply that saw the move).
func movedRecord(f store.KeptFolder, mount string) store.KeptFolder {
left := f.Left
if f.How == store.KeptByWarrant && len(f.Users) > 0 && f.Mount != "" {
left = addLeft(left, f.Mount, f.Placed, f.Fs, f.Users)
}
placed, fs := placeOf(mount)
return store.KeptFolder{Path: f.Path, Mount: mount, Placed: placed, Fs: fs, Want: f.Want, IDs: f.IDs,
Act: f.Act, TakenOn: f.TakenOn, Left: left}
}
// placeOf is the name of the filesystem at mount, and what it was made from.
func placeOf(mount string) (placed, fs string) {
fs = filesystemKey(mount)
return placedFrom(mount, fs), fs
}
// addLeft adds users' entries left on mount, once per mount and uid.
func addLeft(left []store.KeptLeft, mount, placed, fs string, users []store.KeptUser) []store.KeptLeft {
for i := range left {
if left[i].Mount == mount && left[i].Placed == placed {
have := usersOf(left[i].Users)
for _, u := range users {
if _, had := have[u.UID]; !had {
left[i].Users = append(left[i].Users, u)
}
}
return left
}
}
return append(left, store.KeptLeft{Mount: mount, Placed: placed, Fs: fs, Since: time.Now().UTC(),
Users: append([]store.KeptUser(nil), users...)})
}
// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder
// on it is then never kept (it fails closed, and waits).
func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) }
// placedFrom is PlacedID from a filesystem key already read.
func placedFrom(mount, key string) string {
if mount == "" || key == "" {
return ""
}
sum := sha256.Sum256([]byte(mount + "\x00" + key))
return "fs-" + hex.EncodeToString(sum[:6])
}
// filesystemKey is what names the filesystem at mount across reboots, in this order:
//
// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in
// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first;
// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its
// UUID;
// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own
// source string (both report f_fsid 0).
//
// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of
// these answers within statBound: then nothing is kept there. Read once per mount at a time.
func filesystemKey(mount string) string {
if mount == "" {
return ""
}
v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) })
if !ok {
return ""
}
return v.(string)
}
func readFilesystemKey(mount string) string {
if u := MountUUID(mount); u != "" {
return "uuid:" + u
}
if id := MountFsid(mount); strings.Trim(id, "0") != "" {
return "fsid:" + id
}
if source, fstype := mountSourceOf(mount); source != "" && source != "none" {
return "source:" + fstype + ":" + source
}
return ""
}
// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
// server is down blocks a stat for ever, and one access must never hold the apply.
var statBound = 5 * time.Second
// errNoAnswer is a folder whose read did not return within statBound.
var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) }
// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds
// one goroutine per path, never one per apply or look.
type flight struct {
done chan struct{}
v any
}
var inFlight = struct {
sync.Mutex
reads map[string]*flight
}{reads: map[string]*flight{}}
// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not
// answer in time.
func boundedOnce(key string, read func() any) (any, bool) {
inFlight.Lock()
f, running := inFlight.reads[key]
if !running {
f = &flight{done: make(chan struct{})}
inFlight.reads[key] = f
go func() {
f.v = read()
inFlight.Lock()
delete(inFlight.reads, key)
inFlight.Unlock()
close(f.done)
}()
}
inFlight.Unlock()
select {
case <-f.done:
return f.v, true
case <-time.After(statBound):
return nil, false
}
}
func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
type read struct {
info os.FileInfo
err error
}
v, ok := boundedOnce(op+" "+path, func() any {
info, err := stat(path)
return read{info, err}
})
if !ok {
return nil, errNoAnswer
}
r := v.(read)
return r.info, r.err
}
// absPath is an absolute path in words: what pathless takes out.
var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`)
// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other
// path said as "a path".
func Pathless(err error) string { return pathless(err) }
func pathless(err error) string {
var pe *os.PathError
if errors.As(err, &pe) {
return pe.Op + ": " + pe.Err.Error()
}
return absPath.ReplaceAllString(err.Error(), "${1}a path")
}
// recordedMounts is every mount point recorded for another kept folder: where a disk was once seen.
func recordedMounts(except string) []string {
all, err := KeptFolders.All()
if err != nil {
return nil
}
var out []string
for p, f := range all {
if p != except && f.Mount != "" && f.Mount != "/" {
out = append(out, f.Mount)
}
}
return out
}
// RootWords is the reason root's data is stated unhealthy with (ADR 0325 §12): never a path.
func RootWords(why string) string {
return why + ", and root's data is never kept: the operator places the data elsewhere or gives it an owner " +
"(novox/hq ADR 0325 §12)"
}
// applyAccess is one access, applied (ADR 0325): its folder judged, made when it is missing where its data lives,
// and its users' entries kept on it when the mesh may keep it; waiting otherwise, and failing for root's data.
// Nothing is changed on the folder but the entries the mesh adds and the mask that shows them.
func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
in inputs) (Outcome, error) {
// What ADR 0312's grant put an account in is taken back first: no account is in a group for data (§5).
gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed)
out, err := applyAccessFolder(ctx, r, in)
if len(left) > 0 {
out.user, out.groups = previous.User, left
}
if said != "" {
out.Detail = joinDetail(out.Detail, said)
if gave && out.Action == "unchanged" {
// Updated, so what names the access under restart-on — the tool runner — holds the groups it has now.
out.Action = "updated"
}
}
return out, err
}
// applyAccessFolder is applyAccess's folder.
func applyAccessFolder(ctx context.Context, r *declaration.Access, in inputs) (Outcome, error) {
out := begin(r)
path := filepath.Clean(r.Path)
want := in.folders[path]
if !keeps(want, in.identities) {
return legacyAccess(r)
}
users := want.list()
_, name := accessName(r.ID)
rec, _ := keptRecord(path)
v := assessFolder(ctx, path, rec, want != nil && want.taken)
if v.err != nil {
return out, v.err
}
if v.moved {
// What is at the path now is not what was kept (a disk mounted over a folder kept while it was empty
// below it): judged afresh, the record of what was kept there no longer holds.
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
*f = movedRecord(*f, v.mount)
return had
})
}
note := ""
if v.moved {
note = "; what is at it now is on another filesystem than what the mesh kept there, so it was judged afresh"
if rec.How != store.KeptByWarrant {
note += "; the entries on the empty folder kept below are forgotten: hidden under the mount, they grant " +
"nothing on data"
}
}
if len(r.Grant) > 0 {
note += "; its grant is no longer acted on: the mesh puts no account in a group for data (novox/hq ADR 0325 §5)"
}
switch v.kind {
case folderUnknown:
out.Action = "waiting"
out.Detail = v.why + ": the folder is left exactly as it is, and judged again on the next apply" + note
return out, nil
case folderWaitDisk:
out.Action = "waiting"
out.Detail = "the disk that holds it is not there: nothing is made, and nothing mounts it until it is " +
"(novox/hq ADR 0325 §9); waits for the operator to " + DiskWords(name) + note
return out, nil
case folderRoot:
if len(users) == 0 {
out.Action, out.Detail = "unchanged", "present or not, nothing is kept: no identity uses it"+note
return out, nil
}
return out, errors.New(RootWords(v.why) + "; nothing was made or changed")
case folderMake:
made, err := CreatePath(path)
if err != nil {
return out, err
}
out.Action = "created"
out.Detail = fmt.Sprintf("made %d level(s) as its nearest existing parent is: owner, group and mode "+
"(novox/hq ADR 0325 §9)", made)
if len(users) == 0 {
out.Detail += note
return out, nil
}
case folderWaitKeep:
if len(users) == 0 {
out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"+note
return out, nil
}
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Mount, f.Want, f.IDs = v.mount, users, want.ids
f.Placed, f.Fs = placeOf(v.mount)
return true
})
out.Action = "waiting"
out.Detail = "it holds data the mesh has not taken: nothing is changed on it until the operator's warrant " +
"(novox/hq ADR 0325 §7, §8); waits for the operator to " + KeepWords(name) + note
return out, nil
}
if len(users) == 0 {
// Nobody uses it any more: what the mesh added goes on the next walk.
dropping := false
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
f.Want = nil
dropping = len(f.Users) > 0
return had
})
if out.Action == "" {
out.Action, out.Detail = "unchanged", "present; nothing is kept: no identity uses it"
}
if dropping {
out.Detail += "; the entries the mesh added are taken out by the next walk"
}
out.Detail += note
return out, nil
}
changed, err := keepTop(path, usersOf(users))
if errors.Is(err, ErrNoACL) {
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Mount, f.Want = v.mount, users
f.Placed, f.Fs = placeOf(v.mount)
return true
})
out.Action = "waiting"
out.Detail = "its filesystem allows no access entries here: the take act lets it, on the operator's word " +
"(novox/hq ADR 0325 §8); waits for the operator to " + KeepWords(name) + note
return out, nil
}
if err != nil {
return out, err
}
var pending string
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
first := !f.Kept
f.Kept, f.Mount, f.Want, f.IDs = true, v.mount, users, want.ids
f.Placed, f.Fs = placeOf(v.mount)
if v.how == store.KeptByWarrant && f.TakenOn == "" {
// The controller says it was taken, and this engine has no record of where: here, now.
f.TakenOn = PlacedID(v.mount)
}
if first {
f.How, f.At = v.how, time.Now().UTC()
// A folder that held nothing has nothing under it to walk; one the controller says was taken
// without this engine's record is walked whole.
if v.how == store.KeptEmpty || v.how == store.KeptMade {
f.Users = users
}
}
if !sameUsers(f.Users, f.Want) {
pending = "; the entries under it follow on the next walk"
}
return true
})
if out.Action == "" {
out.Action = "unchanged"
if changed {
out.Action = "updated"
}
}
out.Detail = joinDetail(out.Detail, "kept for "+usersWords(users)+": its own entries and default, nothing "+
"else of it changed (novox/hq ADR 0325 §5)"+pending+note)
return out, nil
}
// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes
// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group.
// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next
// apply tries again — and what was said; nothing for an access that granted nothing.
func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted,
handed Handed) (bool, []string, string) {
if previous.User == "" || len(previous.Groups) == 0 {
return false, nil, ""
}
account := previous.User
in, err := system.GroupsOf(ctx, system.Runner(run), account)
if err != nil {
return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+
"back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err)
}
already := setOf(in)
gave := false
var left, said []string
for _, g := range previous.Groups {
switch {
case !already[g]:
continue
case len(wanted.othersAsk(account, g, previous.ID)) > 0:
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g,
strings.Join(wanted.othersAsk(account, g, previous.ID), ", ")))
handed.hand(account, g)
continue
}
gone, why := leaveGroup(ctx, sys, account, g, run)
if !gone {
left = append(left, g)
why += "; tried again on the next apply"
}
gave = gave || gone
said = append(said, why)
}
return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ")
}
// usersWords names users for a person: "mesh-radarr (read-write), the tool runner's nox (read)".
func usersWords(users []store.KeptUser) string {
words := make([]string, 0, len(users))
for _, u := range users {
who := u.Identity
if u.From == declaration.FromToolRunner {
who = "the tool runner's " + who
}
words = append(words, who+" ("+u.Use+")")
}
return strings.Join(words, ", ")
}
// forgetUndeclared marks every recorded folder no declared access names any more: nobody wants its entries, and
// the next walk takes out exactly what the mesh added, then forgets it.
func forgetUndeclared(folders map[string]*folderWant) {
all, err := KeptFolders.All()
if err != nil {
return
}
for path := range all {
if folders[path] != nil {
continue
}
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Want = nil
// A folder never kept, or kept with nobody's entries left anywhere, is forgotten now.
return (f.Kept && len(f.Users) > 0) || len(f.Left) > 0
})
}
}
// MissingPath is an accessed folder that is not there, and its nearest existing ancestor.
type MissingPath struct {
Ancestor string
// Levels are the names to make below Ancestor, outermost first.
Levels []string
// Info is the ancestor as lstat reads it: a link is not a directory.
Info os.FileInfo
}
// NearestAncestor reads, for a path that is not there, its nearest existing ancestor by lstat. An error for a
// path that is there, one not stated exactly, or an ancestor that cannot be read.
func NearestAncestor(path string) (MissingPath, error) {
if err := keptPath(path); err != nil {
return MissingPath{}, err
}
if _, err := lstatWithin(path); err == nil || !errors.Is(err, os.ErrNotExist) {
return MissingPath{}, errors.New("the folder is there, or cannot be read")
}
var levels []string
for p := path; p != "/"; p = filepath.Dir(p) {
levels = append([]string{filepath.Base(p)}, levels...)
parent := filepath.Dir(p)
info, err := lstatWithin(parent)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return MissingPath{}, errors.New("a parent of the folder cannot be read: " + pathless(err))
}
return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil
}
return MissingPath{}, errors.New("no part of the folder's path is there")
}
// accessPaths is every declared access's folder with its id: what no container mounts while it is not there.
//
// Only accesses the mesh keeps (Keeps): an older controller's access holds no container, as before ADR 0325.
func accessPaths(resources []declaration.Resource, refused map[string]error, identities bool) map[string]string {
folders := foldersOf(resources, refused)
out := map[string]string{}
for _, r := range resources {
a, ok := r.(*declaration.Access)
if !ok {
continue
}
if path := filepath.Clean(a.Path); refused[path] == nil && keeps(folders[path], identities) {
out[path] = a.ID
}
}
return out
}
// keeps is whether ADR 0325's keeping applies to a folder: some access naming it carries users, or the
// declaration carries identities — a controller that speaks link.IdentityContract. An older controller's access
// (no users, no identities) is applied exactly as before: mounted when present, refused when absent, nothing kept,
// nothing made, nothing judged (ADR 0051 §7).
func keeps(f *folderWant, identities bool) bool {
return identities || (f != nil && len(f.users) > 0)
}
// Keeps is keeps for a whole declaration's access at path, for the engine's looks.
func Keeps(d *declaration.Declaration, path string) bool {
if d == nil {
return false
}
return keeps(foldersOf(d.Resources, nil)[filepath.Clean(path)], len(d.Identities) > 0)
}
// mountsMissing is the access a container waits for: one whose folder it mounts, or mounts below, is not there
// now — its disk away, or not made yet. The runtime would make it as root (04-ISSUES/026).
func mountsMissing(ctx context.Context, c *declaration.Container, accesses map[string]string) string {
for _, v := range c.Volumes {
src := mountSource(v)
if !strings.HasPrefix(src, "/") {
continue
}
for path, id := range accesses {
if !atOrBelow(filepath.Clean(src), path) {
continue
}
if _, err := statWithin(path); errors.Is(err, os.ErrNotExist) || errors.Is(err, errNoAnswer) {
return id
}
// There, and its disk not: what is at the path is the empty mount point on the filesystem below, and
// a container's writes would land there (§9).
rec, _ := keptRecord(path)
if placedOn(ctx, path, path, rec.Mount, recordedMounts(path)).Away {
return id
}
}
}
return ""
}
// WaitingOnAccess is every container of the declaration that waits for an accessed folder not there, read now:
// not created by the apply, so not judged alive or dead — the access says why.
func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
out := map[string]bool{}
if d == nil {
return out
}
accesses := accessPaths(d.Resources, refusedPlaces(d.Resources), len(d.Identities) > 0)
if len(accesses) == 0 {
return out
}
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && mountsMissing(context.Background(), c, accesses) != "" {
out[c.ID] = true
}
}
return out
}
// The judgements of an access as a look says them.
const (
VerdictKeep = "keep"
VerdictMake = "make"
VerdictWaitKeep = "wait-keep"
VerdictWaitDisk = "wait-disk"
VerdictRoot = "root"
VerdictBroken = "broken"
// VerdictUnknown is a folder whose filesystem could not be named this look: neither kept nor faulted.
VerdictUnknown = "unknown"
// VerdictNobody is a folder no identity uses: nothing is kept, and only a missing disk is said.
VerdictNobody = "nobody"
)
// AccessJudgement is a look's judgement of an access's folder, and why, in words; Placed names the filesystem at
// it for a wait, with no path (PlacedID).
type AccessJudgement struct {
Kind, Why, Placed string
}
// AccessVerdict judges an access's folder for a look, read only (ADR 0325 §7, §9, §12): taken is whether the
// controller says it was taken, used whether any identity uses it. A folder the mesh could keep that no apply has
// kept — its filesystem allowing no ACLs — waits as one not taken does.
func AccessVerdict(ctx context.Context, path string, taken, used bool) AccessJudgement {
rec, _ := keptRecord(path)
v := assessFolder(ctx, path, rec, taken)
switch {
case v.err != nil:
return AccessJudgement{Kind: VerdictBroken, Why: pathless(v.err)}
case v.kind == folderWaitDisk:
return AccessJudgement{Kind: VerdictWaitDisk}
case v.kind == folderUnknown:
return AccessJudgement{Kind: VerdictUnknown, Why: v.why}
case !used:
return AccessJudgement{Kind: VerdictNobody}
case v.kind == folderRoot:
return AccessJudgement{Kind: VerdictRoot, Why: v.why}
case v.kind == folderMake:
return AccessJudgement{Kind: VerdictMake}
case v.kind == folderWaitKeep, v.kind == folderKeep && !rec.Kept && !taken:
return AccessJudgement{Kind: VerdictWaitKeep, Placed: PlacedID(v.mount)}
}
return AccessJudgement{Kind: VerdictKeep}
}
// Leftover is the mesh's entries that could not be taken out yet: under the accesses that last named the folder,
// why, in words with no path.
type Leftover struct {
Module, Access, Name, Path, Why string
}
// Leftovers is every folder the mesh still has entries on that nobody wants (ADR 0325 §8): a folder no access names
// any more whose disk is away, so its drop walk waits; and entries left on a filesystem no longer at a folder. Each
// is said until its drop walk has run.
func Leftovers() []Leftover {
all, err := KeptFolders.All()
if err != nil {
return nil
}
var out []Leftover
for path, f := range all {
module, name, access := recordName(path, f)
dropping := f.Kept && len(f.Want) == 0 && len(f.Users) > 0 && f.Act != nil && f.Act.State == "waiting"
switch {
case dropping:
// Why the walk could not run, as it recorded it: its disk not there, or its filesystem not named.
why := strings.TrimSuffix(strings.TrimPrefix(f.Act.Why, "not walked: "), "; walked once it is")
if why == "" {
why = "the walk could not run"
}
out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " +
name + " could not be removed: " + why + "; they are removed once it can (novox/hq ADR 0325 §8)"})
case len(f.Left) > 0:
out = append(out, Leftover{Module: module, Access: access, Name: name, Path: path, Why: "the mesh's entries for " +
name + " on a filesystem no longer at its folder could not be removed: they are removed once that " +
"filesystem is back (novox/hq ADR 0325 §8)"})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Access < out[j].Access })
return out
}
// recordName is the module, the name and the access a kept folder's record is said under: the accesses that last
// named it, or for a record written before it kept them, a name with no path — the module of a container identity
// among its users, and a folder named by a digest of its path.
func recordName(path string, f store.KeptFolder) (module, name, access string) {
if len(f.IDs) > 0 {
module, name = accessName(f.IDs[0])
return module, name, f.IDs[0]
}
sum := sha256.Sum256([]byte(path))
name = "folder-" + hex.EncodeToString(sum[:4])
for _, u := range append(append([]store.KeptUser(nil), f.Users...), f.Want...) {
if m, ok := strings.CutPrefix(u.Identity, declaration.IdentityPrefix); ok && u.From == declaration.FromContainer {
return m, name, m + "." + name
}
}
return "", name, name
}
// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7).
//
// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container
// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026
// records. So the host checks first and says plainly that the operator must provide the path, rather than
// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports
// a change: the machine did not move, the host merely confirmed a fact about it.
func legacyAccess(r *declaration.Access) (Outcome, error) {
out := begin(r)
info, err := os.Stat(r.Path)
if errors.Is(err, os.ErrNotExist) {
return out, fmt.Errorf(
"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
"nothing, so a missing one is said here rather than made as root by the container "+
"runtime", r.Path)
}
if err != nil {
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf(
"%s is not a directory, and an access is a shared directory the operator provides",
r.Path)
}
mode := declaration.AccessRead
if r.Mode != "" {
mode = r.Mode
}
out.Action = "unchanged"
out.Detail = "operator-owned; present, " + mode + ", nothing managed"
return out, nil
}
internal/apply/keeper_linux.go
+446
View File
@@ -0,0 +1,446 @@
//go:build linux
package apply
import (
"context"
"errors"
"fmt"
"path/filepath"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// The walks under a kept folder, beside the apply (novox/hq ADR 0325 §6, §8, §10).
//
// The take act adds the users' entries to everything under a folder on the operator's warrant; ReconcileKept adds a
// joining user's and removes exactly a dropped user's under a folder already taken; the sweep looks at a bounded
// part of each kept folder on every look, going on where the last stopped, and puts back what an app narrowed.
// One walk at a time per folder: a sweep finds a folder being walked and waits for the next look.
// folderLocks is one lock per kept folder, held by every walk of it.
var folderLocks sync.Map
func folderLock(path string) *sync.Mutex {
l, _ := folderLocks.LoadOrStore(path, &sync.Mutex{})
return l.(*sync.Mutex)
}
// actRecord is a walk's counts as the record keeps them.
func actRecord(act, warrant, by string, started time.Time, w *aclWalk, err error) *store.KeptAct {
c := w.counts
rec := &store.KeptAct{Act: act, Warrant: warrant, By: by, State: "done", Started: started, Ended: time.Now().UTC(),
Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other, Mounts: c.Mounts}
switch {
case errors.Is(err, ErrWalkBound):
rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on at the next apply"
case err != nil:
rec.State, rec.Why = "failed", err.Error()
case w.Failed != "":
rec.Why = fmt.Sprintf("%d entries could not be given their entries: %s", c.Unrepaired, w.Failed)
}
return rec
}
// ReconcileKept walks every kept folder whose entries under it are not what the last apply declared: a user that
// joined has its entries added to everything under it, a dropped user — or one whose use changed — has exactly its
// own taken out (ADR 0325 §8). Bounded by ActLimits; a walk stopped is gone on with at the next call. A folder
// nobody uses any more is forgotten once its entries are out. What it did, in words, for the engine to say.
func ReconcileKept(ctx context.Context) []string {
all, err := KeptFolders.All()
if err != nil {
return []string{"the kept folders could not be read: " + err.Error()}
}
var said []string
for path, f := range all {
if ctx.Err() != nil {
break
}
if len(f.Left) > 0 {
said = append(said, dropLeft(path, f)...)
if f, _ = keptRecord(path); f.Path == "" {
continue
}
}
if !f.Kept || sameUsers(f.Users, f.Want) {
continue
}
said = append(said, reconcileOne(ctx, path, f)...)
}
return said
}
// LeftFor is how long entries left on a filesystem are kept recorded, and said, while it is not back.
var LeftFor = 30 * 24 * time.Hour
// dropLeft takes out the entries the mesh left on a filesystem no longer at the folder, once that filesystem holds
// it again (§8): exactly those users' entries, nothing else. Left on the filesystem the folder is kept on now, they
// are that folder's own, and its walk takes out whichever are no longer wanted. Until then they stay recorded, and
// said (Leftovers).
func dropLeft(path string, f store.KeptFolder) []string {
lock := folderLock(path)
lock.Lock()
defer lock.Unlock()
holding := ""
if _, err := lstatWithin(path); err == nil {
holding = mountHolding(path, MountPoints())
}
holdingID := ""
if holding != "" {
holdingID = PlacedID(holding)
}
var said []string
var still []store.KeptLeft
var mine []store.KeptUser
for _, l := range f.Left {
// On the folder's own filesystem now: the same name, or a name of an earlier build's form at the same
// mount point (which cannot be compared, and is never walked away).
same := l.Mount == f.Mount && (l.Placed == "" || l.Placed == f.Placed || l.Fs == "")
back := l.Mount == holding && (l.Placed == "" || l.Placed == holdingID)
if l.Since.IsZero() {
l.Since = time.Now().UTC()
}
if !same && !back && time.Since(l.Since) > LeftFor {
// **A filesystem not back within LeftFor is gone** (a disk replaced, a pool destroyed): what was left
// on it is forgotten, said once, rather than said unhealthy for ever.
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem not back for %s "+
"are forgotten: it is taken for gone", path, LeftFor))
continue
}
switch {
case f.Kept && same:
// Left on the filesystem the folder is kept on now: the folder's own entries.
mine = append(mine, l.Users...)
case same:
// On the filesystem the folder is on now, not kept yet: never walked away; they are the folder's
// once it is kept there again.
still = append(still, l)
case !back:
still = append(still, l)
default:
drop := map[uint32]bool{}
for _, u := range l.Users {
drop[u.UID] = true
}
w, err := walkKept(path, ActLimits, machineStats, nil, dropVisit(drop))
if err != nil || w.Failed != "" {
still = append(still, l)
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
"were not all taken out, tried again: %s", path, firstOf(pathlessOr(err), w.Failed)))
continue
}
said = append(said, fmt.Sprintf("the kept folder %s: the entries left on a filesystem that is back "+
"were taken out (%d folders and %d files)", path, w.counts.Folders, w.counts.Files))
}
}
_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
k.Left = still
for _, u := range mine {
if _, has := usersOf(k.Users)[u.UID]; !has {
k.Users = append(k.Users, u)
}
}
return had && ((k.Kept && (len(k.Users) > 0 || len(k.Want) > 0)) || len(k.Left) > 0)
})
return said
}
func pathlessOr(err error) string {
if err == nil {
return ""
}
return pathless(err)
}
func firstOf(a, b string) string {
if a != "" {
return a
}
return b
}
func reconcileOne(ctx context.Context, path string, f store.KeptFolder) []string {
lock := folderLock(path)
lock.Lock()
defer lock.Unlock()
// **Judged as an apply judges it, before every walk** (§9, §12): a disk not there leaves its mount point —
// an empty folder of root's on the filesystem below — at the path, and a walk there would write onto root's
// folder, or "drop" entries from a folder that never had them and forget the ones on the data. Nothing is
// walked then, and the record stays for the walk that can reach the data.
if v := assessFolder(ctx, path, f, false); v.err != nil || v.kind != folderKeep || v.moved {
why := "the disk that holds it is not there"
switch {
case v.err != nil:
why = pathless(v.err)
case v.kind == folderRoot:
why = RootWords(v.why)
case v.kind == folderUnknown:
why = v.why
case v.moved || v.kind == folderWaitKeep:
why = "what is at it now is not what the mesh kept there"
}
_ = KeptFolders.Update(path, func(k *store.KeptFolder, had bool) bool {
k.Act = &store.KeptAct{Act: "walk", State: "waiting", Started: time.Now().UTC(),
Why: "not walked: " + why + "; walked once it is"}
return had
})
return []string{fmt.Sprintf("the kept folder %s: not walked, %s", path, why)}
}
have, want := usersOf(f.Users), usersOf(f.Want)
drop := map[uint32]bool{}
adds := false
for uid, rw := range have {
if w, ok := want[uid]; !ok || w != rw {
drop[uid] = true
}
}
for uid, rw := range want {
if h, ok := have[uid]; !ok || h != rw {
adds = true
}
}
act, visit := "add", keepVisit(want)
switch {
case len(drop) > 0 && adds:
visit = dropThenKeep(drop, want)
case len(drop) > 0:
act, visit = "drop", dropVisit(drop)
}
started := time.Now().UTC()
w, err := walkKept(path, ActLimits, machineStats, nil, visit)
rec := actRecord(act, "", "", started, w, err)
_ = KeptFolders.Update(path, func(k *store.KeptFolder, _ bool) bool {
k.Act = rec
// Done only when every entry was reached and changed: one that failed is walked again.
if err == nil && w.Failed == "" {
k.Users = f.Want
}
return k.Kept && (len(k.Want) > 0 || len(k.Users) > 0)
})
return []string{fmt.Sprintf("the kept folder %s: %s walk %s (%d folders and %d files changed, %d had "+
"them)%s", path, act, rec.State, rec.Folders, rec.Files, rec.Kept, orWhy(rec.Why))}
}
func orWhy(why string) string {
if why == "" {
return ""
}
return ": " + why
}
// TakeFolder is the take act (ADR 0325 §8): on the operator's warrant, the users' entries added to the folder of
// the access id and everything under it. It judges the ask against the declaration last applied and the folder as
// it is, sets the filesystem's ACL type where a ZFS dataset allows none yet, and starts the walk: what it answers
// at once, and a channel closed when the walk is through. Refused, nothing is changed.
func TakeFolder(ctx context.Context, d *declaration.Declaration, id, path, warrant, by string,
run Runner) (<-chan struct{}, string, error) {
if d == nil {
return nil, "", errors.New("this engine has applied no declaration, and takes no folder it was not told of")
}
var access *declaration.Access
for _, r := range d.Resources {
if a, ok := r.(*declaration.Access); ok && a.ID == id {
access = a
}
}
switch {
case access == nil:
return nil, "", fmt.Errorf("the access %s is not declared on this machine", id)
case filepath.Clean(access.Path) != path:
return nil, "", fmt.Errorf("the access %s is declared at another folder than the one asked", id)
}
folders := foldersOf(d.Resources, refusedPlaces(d.Resources))
want := folders[path]
users := want.list()
if len(users) == 0 {
return nil, "", errors.New("no identity uses the folder, so there is nothing to keep")
}
rec, _ := keptRecord(path)
v := assessFolder(ctx, path, rec, false)
switch {
case v.err != nil:
return nil, "", v.err
case v.kind == folderWaitDisk:
return nil, "", errors.New("the disk that holds the folder is not there")
case v.kind == folderRoot:
return nil, "", errors.New(RootWords(v.why))
case v.kind == folderMake:
return nil, "", errors.New("the folder is not there; the next apply makes it, and asks nobody")
case v.unnamed:
return nil, "", errors.New(v.why + ", so a warrant could name no filesystem it was given for")
}
lock := folderLock(path)
if !lock.TryLock() {
return nil, "", errors.New("a walk of the folder is running; asked again once it is through, the act goes on")
}
said := ""
_, err := keepTop(path, usersOf(users))
if errors.Is(err, ErrNoACL) {
allowed, aerr := allowACLs(ctx, v.mount, run)
if aerr != nil {
lock.Unlock()
return nil, "", aerr
}
said = allowed + "; "
_, err = keepTop(path, usersOf(users))
}
if err != nil {
lock.Unlock()
return nil, "", err
}
started := time.Now().UTC()
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
if v.moved {
*f = movedRecord(*f, v.mount)
}
if !f.Kept {
f.Kept, f.How, f.At, f.Users = true, store.KeptByWarrant, started, nil
}
f.Warrant, f.By, f.Mount, f.Want, f.IDs = warrant, by, v.mount, users, want.ids
// The warrant is for the filesystem it was carried out on (§8), named as a wait names it.
f.Placed, f.Fs = placeOf(v.mount)
f.TakenOn = f.Placed
f.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, State: "running", Started: started}
return true
})
done := make(chan struct{})
go func() {
defer close(done)
defer lock.Unlock()
w, err := walkKept(path, ActLimits, machineStats, nil, keepVisit(usersOf(users)))
rec := actRecord("take", warrant, by, started, w, err)
_ = KeptFolders.Update(path, func(f *store.KeptFolder, _ bool) bool {
f.Act = rec
if err == nil {
f.Users = users
}
return true
})
}()
return done, said + "the take act started under the warrant " + warrant + ": the entries of " +
usersWords(users) + " are added to everything under the folder, following no link and entering no other " +
"filesystem", nil
}
// allowACLs lets the filesystem at mount carry ACLs, inside the take act only (ADR 0325 §8): a ZFS dataset's
// acltype set to posixacl, the dataset read from the machine's mounts. Any other filesystem is refused, naming why.
func allowACLs(ctx context.Context, mount string, run Runner) (string, error) {
source, fstype := mountSourceOf(mount)
if fstype != "zfs" || source == "" {
return "", fmt.Errorf("the filesystem that holds the folder (%s) allows no access entries, and only a ZFS "+
"dataset's can be let by the act; the operator mounts it with ACLs", orUnknown(fstype))
}
if _, err := run(ctx, "zfs", "set", "acltype=posixacl", source); err != nil {
return "", fmt.Errorf("the ZFS dataset that holds the folder could not be let carry access entries "+
"(zfs set acltype=posixacl): %w", err)
}
return "set acltype=posixacl on the ZFS dataset " + source, nil
}
func orUnknown(s string) string {
if s == "" {
return "of a type the machine does not say"
}
return s
}
// KeptLook is what one look found of a kept folder (ADR 0325 §10).
type KeptLook struct {
// Short is every user the folder itself gives less than its right, by entry and mask together, in words.
Short []string
// Busy is a walk of the folder running: the sweep waited.
Busy bool
// Sweep is the sweep's place after this look.
Sweep store.KeptSweep
// Repaired is how many entries this look found narrowed and put back; Unrepaired how many it could not.
Repaired, Unrepaired int
// Failed is why the sweep could not look, in words with no path.
Failed string
}
// LookKept looks at a kept folder once: the folder itself judged for each user, then the sweep's bounded part of
// what is under it, from where the last look stopped, putting back what an app narrowed. The sweep's place is
// recorded; a look that ran out of its bound says so.
func LookKept(path string, users []store.KeptUser, limits WalkLimits) KeptLook {
var out KeptLook
short, err := judgeTop(path, users)
if err != nil {
out.Failed = err.Error()
return out
}
out.Short = short
lock := folderLock(path)
if !lock.TryLock() {
out.Busy = true
rec, _ := keptRecord(path)
out.Sweep = rec.Sweep
return out
}
defer lock.Unlock()
rec, _ := keptRecord(path)
sw := rec.Sweep
w, err := walkKept(path, limits, machineStats, sw.Cursor, keepVisit(usersOf(users)))
out.Repaired, out.Unrepaired = w.counts.Narrowed-w.counts.Unrepaired, w.counts.Unrepaired
sw.Read += w.counts.Read
sw.Narrowed += w.counts.Narrowed
sw.Unrepaired += w.counts.Unrepaired
switch {
case errors.Is(err, ErrWalkBound):
sw.Cursor, sw.Bound = w.Cursor, true
case err != nil:
out.Failed = err.Error()
default:
// Covered whole: the next look starts a new pass, and what this one could not put back is kept until
// the next pass is through.
sw = store.KeptSweep{Covered: time.Now().UTC(), LastUnrepaired: sw.Unrepaired}
}
out.Sweep = sw
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
f.Sweep = sw
return had
})
return out
}
// judgeTop is every user the folder itself gives less than its right, read only.
func judgeTop(path string, users []store.KeptUser) ([]string, error) {
var short []string
_, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
if err != nil {
return nil, err
}
defer unix.Close(fd)
a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777)
if err != nil {
return nil, err
}
for _, u := range users {
need := needOn(u.Use == declaration.AccessReadWrite, true)
if got := effective(a, u.UID, st.Uid, uint32(st.Mode)&0o7777); got&need != need {
short = append(short, fmt.Sprintf("%s has %s of the %s it needs", u.Identity, permWords(got),
permWords(need)))
}
}
return short, nil
}
// JudgeKept reads, without changing anything, how many entries under a kept folder give a user less than its
// right, within limits: what the access says while an entry an app narrowed is not put back yet (ADR 0325 §6).
func JudgeKept(path string, users []store.KeptUser, limits WalkLimits) (WalkCounts, error) {
um := usersOf(users)
w, err := walkKept(path, limits, machineStats, nil, func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool,
error) {
a, err := getACL(fd, aclAccessXattr, uint32(st.Mode)&0o7777)
if err != nil {
return false, false, false, err
}
n, u := judgeACL(a, um, st, folder)
return false, n, u, nil
})
return w.counts, err
}
internal/apply/kept_test.go
+1523
View File
File diff suppressed because it is too large Load Diff
internal/apply/mesh_names_test.go
+3 -3
View File
@@ -23,7 +23,7 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
Name: was.Name, Image: was.Image,
Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"},
}
if containerSpecReading(was, nil, nil, nil) == containerSpecReading(moved, nil, nil, nil) {
if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) {
t.Fatal("a container whose mesh names moved compares equal, so it is never recreated")
}
@@ -33,14 +33,14 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
Name: moved.Name, Image: moved.Image,
Hosts: []string{moved.Hosts[1], moved.Hosts[0]},
}
if containerSpecReading(moved, nil, nil, nil) != containerSpecReading(reordered, nil, nil, nil) {
if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) {
t.Fatal("the same names in another order read as a different container")
}
// A container the mesh gives no names is unaffected, so nothing is recreated for a field it
// does not set.
plain := &declaration.Container{Name: "plex", Image: was.Image}
if containerSpecReading(plain, nil, nil, nil) == containerSpecReading(was, nil, nil, nil) {
if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) {
return // different for other reasons, which is fine
}
}
internal/apply/oldshape_test.go
+136
View File
@@ -0,0 +1,136 @@
//go:build linux
package apply
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// The engine delivers before the controller half (novox/hq ADR 0325): a controller that does not speak
// link.IdentityContract sends no identities, no users and no taken, and every access of its declarations is applied
// exactly as before this engine — mounted when present, refused when absent (ADR 0051 §7), nothing kept, nothing
// made, no container held, nothing judged. These are origin/main's outcomes, word for word.
func TestAnOlderControllersAccessesAreAppliedExactlyAsBefore(t *testing.T) {
keeping(t, nil, nil)
dir := userDir(t)
unwritable := filepath.Join(dir, "library") // its group may not write it
empty := filepath.Join(dir, "spool")
absent := filepath.Join(dir, "media", "movies")
for _, d := range []string{unwritable, empty} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
if err := os.Chmod(d, 0o755); err != nil {
t.Fatal(err)
}
}
// A root-owned empty folder, read only.
rootEmpty := ""
for _, c := range []string{"/var/empty", "/srv/http", "/srv/ftp"} {
if entries, err := os.ReadDir(c); err == nil && len(entries) == 0 {
if uid, _, _ := statOwner(mustLstat(t, c)); uid == 0 {
rootEmpty = c
break
}
}
}
resources := []string{
`{"id":"radarr.library","type":"access","path":"` + unwritable + `","mode":"read-write"}`,
`{"id":"sab.spool","type":"access","path":"` + empty + `","mode":"read-write"}`,
`{"id":"plex.movies","type":"access","path":"` + absent + `"}`,
`{"id":"plex.app","type":"container","name":"plex","image":"` + pinned + `","volumes":["` + absent + `:/movies"]}`,
}
if rootEmpty != "" {
resources = append(resources, `{"id":"web.root","type":"access","path":"`+rootEmpty+`","mode":"read-write"}`)
}
var started bool
run := func(_ context.Context, name string, args ...string) (string, error) {
if len(args) > 0 && args[0] == "run" {
started = true
}
if len(args) > 0 && args[0] == "container" {
if started {
return "true\t" + containerSpec(declare(t, resources[3]).Resources[0].(*declaration.Container), inputs{}), nil
}
return "", errors.New("no such container")
}
return "", nil
}
toUser(t)
report, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")), store.State{},
store.OriginDeclared, run, nil, nil)
if err == nil || !strings.Contains(err.Error(), absent+" is not there, and the mesh does not own it — the operator "+
"must provide it") {
t.Fatalf("an absent access is not refused as before: %v", err)
}
for id, mode := range map[string]string{"radarr.library": "read-write", "sab.spool": "read-write"} {
o := outcomeOf(report, id)
if o.Action != "unchanged" || o.Detail != "operator-owned; present, "+mode+", nothing managed" {
t.Errorf("%s: %+v, and before it was unchanged, %q", id, o, "operator-owned; present, "+mode+", nothing managed")
}
}
if rootEmpty != "" {
if o := outcomeOf(report, "web.root"); o.Action != "unchanged" {
t.Errorf("a root-owned folder was judged: %+v", o)
}
}
if !started {
t.Error("a container mounting an absent access was held, and before it was created")
}
if _, err := os.Stat(absent); !errors.Is(err, os.ErrNotExist) {
t.Error("an absent access was made")
}
for _, d := range []string{unwritable, empty} {
if n, _ := unix.Getxattr(d, aclAccessXattr, nil); n > 0 {
t.Errorf("an ACL was set on %s", d)
}
if info := mustLstat(t, d); info.Mode().Perm() != 0o755 {
t.Errorf("%s changed mode: %o", d, info.Mode().Perm())
}
}
if all, _ := KeptFolders.All(); len(all) != 0 {
t.Errorf("folders were recorded as kept: %v", all)
}
if len(Leftovers()) != 0 || len(WaitingOnAccess(declare(t, strings.Join(resources, ",")))) != 0 {
t.Error("something was said or held for an older controller's access")
}
}
func mustLstat(t *testing.T, path string) os.FileInfo {
t.Helper()
info, err := os.Lstat(path)
if err != nil {
t.Fatal(err)
}
return info
}
// Review (low): a folder no access names, whose filesystem could not be named this look, is not said as its disk
// being away.
func TestALeftoverSaysWhyItsWalkCouldNotRun(t *testing.T) {
uuid, fsid, source := "6f1c0a4e-1b2d-4c3e-8f00-aa11bb22cc33", "0000080100000000", "/dev/sdb1"
_, lib, access := poolNamed(t, &uuid, &fsid, &source)
takeIt(t, lib, access)
other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(filepath.Dir(lib)), "x") + `"}`
if _, _, err := applyAccesses(t, store.State{}, other); err != nil {
t.Fatal(err)
}
uuid, fsid, source = "", "0000000000000000", "" // nothing names it now
toUser(t)
ReconcileKept(context.Background())
left := Leftovers()
if len(left) != 1 || strings.Contains(left[0].Why, "disk") || !strings.Contains(left[0].Why, "could not be named") {
t.Fatalf("the leftover does not say why its walk could not run: %+v", left)
}
}
internal/apply/placed.go
+261
View File
@@ -0,0 +1,261 @@
package apply
import (
"bufio"
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
)
// Whether a shared folder's disk is there (novox/hq ADR 0325 §9).
//
// **The rule the engine keeps.** A folder is placed on the filesystem it is first seen on: the mount point that
// holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the
// kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged
// against that record: held by another mount point than the one recorded while the recorded one is not mounted is
// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is
// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made
// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's
// word as any folder holding data does (ADR 0325 §8).
//
// **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be:
// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names
// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every
// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point
// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what
// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list`
// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen
// empty below it is kept as empty, and the move above takes it back the moment the disk is there.
// MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test
// can stand in for mounts it cannot make.
var MountPoints = func() []string {
return readMountPoints()
}
// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history").
var mountHistory = struct {
sync.Mutex
seen map[string]bool
}{seen: map[string]bool{}}
// sawMounted adds points to the history, and is the history: every mount point seen, never "/".
func sawMounted(points []string) []string {
mountHistory.Lock()
defer mountHistory.Unlock()
for _, p := range points {
if p != "/" {
mountHistory.seen[p] = true
}
}
out := make([]string, 0, len(mountHistory.seen))
for p := range mountHistory.seen {
out = append(out, p)
}
return out
}
func readMountPoints() []string {
raw, err := os.ReadFile("/proc/self/mountinfo")
if err != nil {
return []string{"/"}
}
return mountinfoPoints(string(raw))
}
// MountTargets is every place a disk is meant to be mounted: /etc/fstab's targets and ZFS datasets' mount points,
// read at most once a minute — every look judges every access by them. A variable so a test can stand in.
var MountTargets = func(ctx context.Context) []string {
targets.Lock()
defer targets.Unlock()
if targets.read.IsZero() || time.Since(targets.read) > time.Minute {
targets.list, targets.read = readMountTargets(ctx), time.Now()
}
return append([]string(nil), targets.list...)
}
var targets struct {
sync.Mutex
list []string
read time.Time
}
func readMountTargets(ctx context.Context) []string {
var out []string
if raw, err := os.ReadFile("/etc/fstab"); err == nil {
out = append(out, fstabTargets(string(raw))...)
}
out = append(out, mountUnitTargets(systemdUnitDirs)...)
if _, err := exec.LookPath("zfs"); err == nil {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if raw, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "mountpoint").Output(); err == nil {
for _, line := range strings.Split(string(raw), "\n") {
if p := strings.TrimSpace(line); strings.HasPrefix(p, "/") {
out = append(out, filepath.Clean(p))
}
}
}
}
return out
}
// systemdUnitDirs are where systemd keeps the units a machine mounts by.
var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator",
"/usr/lib/systemd/system", "/lib/systemd/system"}
// mountUnitTargets reads the `Where=` of every .mount unit in dirs.
func mountUnitTargets(dirs []string) []string {
var out []string
for _, dir := range dirs {
names, err := filepath.Glob(filepath.Join(dir, "*.mount"))
if err != nil {
continue
}
for _, name := range names {
raw, err := os.ReadFile(name)
if err != nil {
continue
}
for _, line := range strings.Split(string(raw), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") {
out = append(out, filepath.Clean(strings.TrimSpace(v)))
}
}
}
}
return out
}
// mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes
// undone.
func mountinfoPoints(raw string) []string {
var out []string
for _, line := range strings.Split(raw, "\n") {
fields := strings.Fields(line)
if len(fields) < 5 {
continue
}
out = append(out, filepath.Clean(unescapeMount(fields[4])))
}
return out
}
// mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its
// name. Empty when the machine says nothing of it.
func mountSourceOf(point string) (source, fstype string) { return MountSource(point) }
// MountSource is mountSourceOf as the machine answers it. A variable so a test can stand in for another filesystem
// mounted at the same point.
var MountSource = func(point string) (source, fstype string) {
raw, err := os.ReadFile("/proc/self/mountinfo")
if err != nil {
return "", ""
}
for _, line := range strings.Split(string(raw), "\n") {
before, after, ok := strings.Cut(line, " - ")
fields, rest := strings.Fields(before), strings.Fields(after)
if !ok || len(fields) < 5 || len(rest) < 2 {
continue
}
if filepath.Clean(unescapeMount(fields[4])) == point {
source, fstype = unescapeMount(rest[1]), rest[0]
}
}
return source, fstype
}
// fstabTargets reads the second field of each fstab line that mounts somewhere: not swap, not a comment.
func fstabTargets(raw string) []string {
var out []string
sc := bufio.NewScanner(strings.NewReader(raw))
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || !strings.HasPrefix(fields[1], "/") {
continue
}
out = append(out, filepath.Clean(unescapeMount(fields[1])))
}
return out
}
// unescapeMount undoes the kernel's and fstab's octal escapes: \040 is a space.
func unescapeMount(s string) string {
if !strings.Contains(s, `\`) {
return s
}
var b strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+4 <= len(s) {
if v, ok := octal3(s[i+1 : i+4]); ok {
b.WriteByte(v)
i += 3
continue
}
}
b.WriteByte(s[i])
}
return b.String()
}
func octal3(s string) (byte, bool) {
if len(s) != 3 {
return 0, false
}
v := 0
for _, c := range s {
if c < '0' || c > '7' {
return 0, false
}
v = v*8 + int(c-'0')
}
return byte(v), v < 256
}
// mountHolding is the longest of points that is path or above it: the mount point that holds it.
func mountHolding(path string, points []string) string {
best := "/"
for _, p := range points {
if atOrBelow(path, p) && len(p) > len(best) {
best = p
}
}
return best
}
// placement is the judgement of where a folder's data lives.
type placement struct {
// Mount is the mount point that holds what is there: the folder, or its nearest existing parent.
Mount string
// Away is that the disk the folder was placed on is not there.
Away bool
// Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what
// was kept.
Moved bool
}
// placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent),
// against what was recorded and where disks are meant to be.
func placedOn(ctx context.Context, path, existing, recorded string, others []string) placement {
points := MountPoints()
mounted := map[string]bool{}
for _, p := range points {
mounted[p] = true
}
have := mountHolding(existing, points)
seen := sawMounted(points)
if recorded != "" {
return placement{Mount: have, Away: have != recorded && !mounted[recorded],
Moved: have != recorded && mounted[recorded]}
}
for _, t := range append(append(MountTargets(ctx), others...), seen...) {
if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] {
return placement{Mount: have, Away: true}
}
}
return placement{Mount: have}
}
internal/apply/plan.go
+1 -1
View File
@@ -112,7 +112,7 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
step.Verb, step.Why = "restore", "no longer declared; the original the mesh wrote over goes back "+
"from "+orphan.Kept+", unless the file was changed since the mesh last wrote it"
case orphan.Type == string(declaration.TypeAccess) && orphan.User != "" && len(orphan.Groups) > 0:
// In removeAccess's words (novox/hq ADR 0312 §5).
// In takeBackGranted's words (novox/hq ADR 0325 §5: no account is in a group for data).
step.Verb, step.Why = "restore", "no longer declared; the path is left as it is, and "+orphan.User+
" leaves "+strings.Join(orphan.Groups, ", ")+", which the mesh put it in, unless another declared "+
"resource still asks for it"
internal/apply/reads_test.go
+1 -1
View File
@@ -212,7 +212,7 @@ func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *te
t.Fatal(err)
}
d := declare("5432")
legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil, nil)
legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil)
known := store.State{}
known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env,
Wrote: digestOf("DATABASE_PORT=5432\n")})
internal/apply/roots.go
+147
View File
@@ -0,0 +1,147 @@
package apply
import (
"context"
"fmt"
"os"
"os/exec"
"os/user"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/accounts"
)
// Root's numbers, which no module identity is and whose data is never kept (novox/hq ADR 0325 §11, §12).
//
// A module identity whose uid or gid is 0, or whose gid is a group whose members act as root (accounts.RootGroups,
// the account judge's own closed list), would be a way to root through a container or a shared folder: refused,
// naming why. A shared folder owned by root or such a group is the machine's own data, never kept.
// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine
// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine
// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number
// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and
// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can
// stand in.
var RootGroupIDs = func() map[int]string {
rootIDs.Lock()
defer rootIDs.Unlock()
if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute {
rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now()
}
return rootIDs.ids
}
var rootIDs struct {
sync.Mutex
ids map[int]string
at time.Time
}
func readRootGroupIDs() map[int]string {
out := map[int]string{}
if raw, err := os.ReadFile("/etc/group"); err == nil {
for name, gid := range groupEntries(string(raw)) {
if slices.Contains(accounts.RootGroups, name) {
out[gid] = name
}
}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
for _, name := range accounts.RootGroups {
raw, err := exec.CommandContext(ctx, "getent", "group", name).Output()
if err != nil {
continue
}
for found, gid := range groupEntries(string(raw)) {
if found == name {
out[gid] = name
}
}
}
return out
}
// groupEntries reads group database lines, name:password:gid:members, as name to number.
func groupEntries(raw string) map[string]int {
out := map[string]int{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Split(strings.TrimSpace(line), ":")
if len(fields) < 3 || strings.HasPrefix(fields[0], "#") {
continue
}
if gid, err := strconv.Atoi(fields[2]); err == nil {
out[fields[0]] = gid
}
}
return out
}
// GroupName is the name this machine's group database gives a number, empty when none. A variable so a test can
// stand in for groups it cannot make.
var GroupName = func(gid int) string {
if g, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil {
return g.Name
}
return ""
}
// rootGroup is why gid is a root group here — 0, a number a root group has, or a number named as one — or "".
func rootGroup(gid int) string {
if gid == 0 {
return "group 0"
}
if name, is := RootGroupIDs()[gid]; is {
return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name)
}
if name := GroupName(gid); slices.Contains(accounts.RootGroups, name) {
return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name)
}
return ""
}
// rootOwned is why a folder owned by uid and gid is root's data (ADR 0325 §12), or "": owned by root, or by a
// root group.
func rootOwned(uid, gid uint32) string {
if uid == 0 {
return "owned by root"
}
if why := rootGroup(int(gid)); why != "" {
return "owned by " + why
}
return ""
}
// rootIdentity is why a module identity's numbers are refused (ADR 0325 §11), or nil.
func rootIdentity(uid, gid int) error {
if uid <= 0 {
return fmt.Errorf("its uid is %d, and a module's identity is never root (novox/hq ADR 0325 §11)", uid)
}
if why := rootGroup(gid); why != "" {
return fmt.Errorf("its gid is %s, and a module's identity is never in a root group (novox/hq ADR 0325 §11)", why)
}
return nil
}
// Handed is every group a record that went in this apply put an account in, and another declared resource still
// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group
// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it
// there went first (novox/hq ADR 0252 §3).
type Handed map[string]map[string]bool
func (h Handed) hand(account, group string) {
if h == nil {
return
}
if h[account] == nil {
h[account] = map[string]bool{}
}
h[account][group] = true
}
func (h Handed) has(account, group string) bool { return h[account][group] }
internal/apply/schedule.go
+2 -15
View File
@@ -73,9 +73,6 @@ type scheduledJob struct {
// hold is the runtime names of the containers held still for the duration of a run, in the
// order the step named them (novox/hq ADR 0189).
hold []string
// groups is the number of every group its read-write accesses grant it, read when it was armed (novox/hq
// ADR 0312): a fire runs with them as an apply's container would.
groups []int
}
// heldStillFor is the runtime names of the containers a step holds still, resolved from ids.
@@ -134,14 +131,6 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
defer s.mu.Unlock()
seen := map[string]bool{}
// The groups each step's read-write accesses grant (novox/hq ADR 0312), read once, only when a step is here.
var granted map[string][]int
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && c.Schedule != "" {
granted = ContainerGroupsOf(d)
break
}
}
for _, r := range d.Resources {
c, ok := r.(*declaration.Container)
if !ok || c.Schedule == "" {
@@ -169,7 +158,7 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
// Nothing to read: a scheduled container may not declare restart-on — it runs to completion
// on its cadence rather than staying running to be restarted — so its identity cannot
// depend on another resource's content and there is nothing to pass.
spec := containerSpec(c, inputs{containerGroups: granted})
spec := containerSpec(c, inputs{})
// The runtime stops containers by name; the declaration names them by id. Resolved here,
// against the declaration this job was armed from, so a fire never has to look anything up
// (novox/hq ADR 0189). The parser has already refused an id that is not a container here.
@@ -178,14 +167,12 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
// Unchanged: keep where it is in its cadence, refresh the declaration pointer only.
existing.container = c
existing.hold = hold
existing.groups = granted[c.Identity()]
continue
}
// New or changed: arm it for the next due minute after now.
next, _ := cron.Next(s.clock.Now())
s.jobs[c.Identity()] = &scheduledJob{
id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold,
groups: granted[c.Identity()],
}
}
@@ -295,7 +282,7 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) {
// one that is not there is the state we want, so its error is ignored — the same as run-once.
_, _ = s.run(ctx, cri, "rm", "-f", j.container.Name)
args := foregroundRunArgs(j.container, j.spec, j.groups)
args := foregroundRunArgs(j.container, j.spec)
if _, err := s.run(ctx, cri, args...); err != nil {
s.log(fmt.Sprintf(
"scheduled step %s: run exited non-zero — recorded, and left for the next run "+
internal/apply/vocabulary_test.go
+3 -3
View File
@@ -447,11 +447,11 @@ func TestAChangedResolverOrAddressIsAChangedContainer(t *testing.T) {
base := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00"}
withDns := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", Dns: []string{"192.168.203.254"}}
withIP := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", IP: "192.168.203.254"}
plain := containerSpecReading(base, nil, nil, nil)
if containerSpecReading(withDns, nil, nil, nil) == plain {
plain := containerSpecReading(base, nil, nil)
if containerSpecReading(withDns, nil, nil) == plain {
t.Error("adding a resolver did not change the spec, so it can never reach a running container")
}
if containerSpecReading(withIP, nil, nil, nil) == plain {
if containerSpecReading(withIP, nil, nil) == plain {
t.Error("adding an address did not change the spec, so it can never reach a running container")
}
}
internal/apply/walk.go
+66
View File
@@ -0,0 +1,66 @@
package apply
import (
"errors"
"path/filepath"
"time"
)
// A bounded walk under a kept folder (novox/hq ADR 0325 §8, §10, issue 567).
//
// **Walked by descriptor.** The folder is opened as a mesh directory is (meshWalk: no link another account could
// make, the folder itself no link), and everything under it is reached from its parent's descriptor with
// O_NOFOLLOW: a link met is passed by, a folder whose device is not the kept folder's is not entered, and an entry
// opened is checked to be the one that was looked at. An entry's ACL is read and written through its own
// descriptor, so a name swapped for a link between the look and the change changes nothing.
//
// **Bounded, and resumed.** A media library is millions of files, and a walk that never ends is a machine that
// cannot say what it did. Each walk reads at most so many entries for at most so long; one that meets its bound
// says how far it got (a cursor, the names of each level below the folder, kept in the engine's state and never
// said) and the next goes on from there, in the order of the names, so successive looks cover a folder whole.
// WalkLimits bound one walk under a folder: how many entries it reads, and for how long.
type WalkLimits struct {
Entries int
Time time.Duration
}
// ActLimits bound the take act and the walk that adds or drops a user's entries. One stopped goes on at the next
// apply, from the start: what already has its entries is passed quickly.
var ActLimits = WalkLimits{Entries: 5_000_000, Time: 30 * time.Minute}
// SweepLimits bound the sweep of one kept folder on one look (ADR 0325 §10): the look must come back within its
// own bound whatever the folder holds.
var SweepLimits = WalkLimits{Entries: 2000, Time: 2 * time.Second}
// WalkCounts is what one walk did or found. Counts only: never a path below the folder.
type WalkCounts struct {
// Folders and Files are the entries changed; Kept the ones that already had what they need.
Folders, Files, Kept int
// Links are links passed by, Other entries neither a file nor a folder, Mounts folders on another
// filesystem not entered.
Links, Other, Mounts int
// Read is every entry looked at.
Read int
// Narrowed is the entries that gave a user less than its right (judged by entry and mask together) and
// were put back; Unrepaired the ones that could not be — the owner's bits, which the mesh never changes.
Narrowed, Unrepaired int
}
// ErrWalkBound is a walk that met its bound before it was through: what it did stays, and the cursor says where
// the next one goes on.
var ErrWalkBound = errors.New("the walk met its bound")
// ErrNoACL is a filesystem that allows no ACLs here (ENOTSUP): a ZFS dataset whose acltype is off, say. The access
// waits for the take act, which sets it where it can (ADR 0325 §8).
var ErrNoACL = errors.New("the filesystem allows no access entries")
// keptPath refuses a folder that is not stated exactly: absolute and clean, so `..` and a doubled separator
// never name it.
func keptPath(path string) error {
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
return errors.New("the folder is not stated as the engine states one (absolute, no `..`, no doubled or " +
"trailing separator); nothing was changed")
}
return nil
}
internal/apply/walk_linux.go
+572
View File
@@ -0,0 +1,572 @@
//go:build linux
package apply
// The descriptor half of walk.go and acl.go (novox/hq ADR 0325 §5, §8, §9, §10).
import (
"errors"
"fmt"
"io"
"os"
"sort"
"time"
"golang.org/x/sys/unix"
)
// walkStats is how a walk reads an entry and an open descriptor: the machine's own calls, or a test's stand-in
// for a device it cannot mount.
type walkStats struct {
at func(dirfd int, name string, st *unix.Stat_t) error
fd func(fd int, st *unix.Stat_t) error
}
var machineStats = walkStats{
at: func(dirfd int, name string, st *unix.Stat_t) error {
return unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW)
},
fd: unix.Fstat,
}
// visitFunc is what a walk does to one folder or file, open on fd, whose stat is st: whether it changed it, and
// whether it found the entry narrowed and could not put it back. An error stops the walk.
type visitFunc func(fd int, st unix.Stat_t, folder bool) (changed, narrowed, unrepaired bool, err error)
// aclWalk is one walk under a kept folder, from the descriptor of the folder itself.
type aclWalk struct {
dev uint64
deadline time.Time
limit int
stats walkStats
visit visitFunc
counts WalkCounts
// at is the names of the levels below the folder down to the entry being walked; Cursor, once the walk met
// its bound, where the next goes on.
at []string
Cursor []string
// Failed is the first failure on one entry, in words with no path.
Failed string
}
// openKept opens a kept folder as a mesh directory is opened (no link another account could make, the folder
// itself no link) and starts a walk from it on the folder's own device.
func openKept(path string, limits WalkLimits, stats walkStats, visit visitFunc) (*aclWalk, int, unix.Stat_t, error) {
var st unix.Stat_t
if err := keptPath(path); err != nil {
return nil, -1, st, err
}
mw := &meshWalk{want: path}
fd, err := mw.open()
if errors.Is(err, errNotThereYet) {
return nil, -1, st, errors.New("the folder is not there; nothing was changed")
}
if err != nil {
return nil, -1, st, errors.New("the folder is not walked: " + pathless(err))
}
if mw.At != path {
unix.Close(fd)
return nil, -1, st, errors.New("the folder is reached through a link, and the engine walks no folder " +
"through one; nothing was changed")
}
if err := stats.fd(fd, &st); err != nil {
unix.Close(fd)
return nil, -1, st, fmt.Errorf("the folder cannot be read: %v; nothing was changed", err)
}
return &aclWalk{dev: uint64(st.Dev), deadline: time.Now().Add(limits.Time), limit: limits.Entries,
stats: stats, visit: visit}, fd, st, nil
}
// walkKept walks path whole, or from resume, with visit: the walk, with what it did and the cursor where it
// stopped when it met its bound (with ErrWalkBound), and the walk's own error otherwise.
func walkKept(path string, limits WalkLimits, stats walkStats, resume []string, visit visitFunc) (*aclWalk, error) {
w, fd, st, err := openKept(path, limits, stats, visit)
if err != nil {
return &aclWalk{}, err
}
return w, w.dir(fd, st, resume)
}
// read counts one entry, and says the bound when it is met, keeping where the walk stood as the cursor.
func (w *aclWalk) read() error {
if w.counts.Read >= w.limit || time.Now().After(w.deadline) {
w.Cursor = append([]string(nil), w.at...)
return ErrWalkBound
}
w.counts.Read++
return nil
}
// count adds one visit's result.
func (w *aclWalk) count(folder, changed, narrowed, unrepaired bool) {
switch {
case changed && folder:
w.counts.Folders++
case changed:
w.counts.Files++
default:
w.counts.Kept++
}
if narrowed {
w.counts.Narrowed++
}
if unrepaired {
w.counts.Unrepaired++
}
}
// dir is one folder, open on fd, whose stat is st: the folder itself (unless the walk resumes below it, which an
// earlier walk visited), then every entry in it in the order of their names. It closes fd.
func (w *aclWalk) dir(fd int, st unix.Stat_t, resume []string) error {
f := os.NewFile(uintptr(fd), "")
defer f.Close()
if len(resume) == 0 {
if err := w.read(); err != nil {
return err
}
if err := w.visited(fd, st, true); err != nil {
return err
}
}
names, err := f.Readdirnames(-1)
if err != nil {
return fmt.Errorf("a folder could not be read: %v", err)
}
sort.Strings(names)
for _, name := range names {
var below []string
if len(resume) > 0 {
if name < resume[0] {
continue
}
if name == resume[0] {
below = resume[1:]
}
resume = nil
}
w.at = append(w.at, name)
err := w.entry(fd, name, below)
w.at = w.at[:len(w.at)-1]
if err != nil {
return err
}
}
return nil
}
// entry is one name in the folder open on dirfd: a link passed by, a folder on this device walked, a file
// visited through its own descriptor, anything else left.
func (w *aclWalk) entry(dirfd int, name string, resume []string) error {
var st unix.Stat_t
if err := w.stats.at(dirfd, name, &st); err != nil {
if errors.Is(err, unix.ENOENT) {
return nil
}
return fmt.Errorf("an entry could not be read: %v", err)
}
switch st.Mode & unix.S_IFMT {
case unix.S_IFLNK:
w.counts.Links++
return nil
case unix.S_IFDIR, unix.S_IFREG:
default:
w.counts.Other++
return nil
}
if uint64(st.Dev) != w.dev {
w.counts.Mounts++
return nil
}
isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR
flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK
if isDir {
flags |= unix.O_DIRECTORY
}
fd, err := unix.Openat(dirfd, name, flags, 0)
switch {
case errors.Is(err, unix.ENOENT):
return nil
case errors.Is(err, unix.ELOOP), errors.Is(err, unix.ENOTDIR):
// Swapped for a link, or for something else, after it was looked at: passed by.
w.counts.Links++
return nil
case err != nil:
return fmt.Errorf("an entry could not be opened: %v", err)
}
var opened unix.Stat_t
if err := w.stats.fd(fd, &opened); err != nil {
unix.Close(fd)
return fmt.Errorf("an entry could not be read: %v", err)
}
if opened.Dev != st.Dev || opened.Ino != st.Ino || opened.Mode&unix.S_IFMT != st.Mode&unix.S_IFMT {
unix.Close(fd)
return errors.New("an entry changed while it was walked, and the walk stopped there")
}
if isDir {
return w.dir(fd, opened, resume)
}
defer unix.Close(fd)
if err := w.read(); err != nil {
return err
}
return w.visited(fd, opened, false)
}
// visited visits one entry and counts what it did. A filesystem that allows no ACLs stops the walk; any other
// failure on one entry (an immutable file, say) is counted unrepaired, its words kept once, and the walk goes on.
func (w *aclWalk) visited(fd int, st unix.Stat_t, folder bool) error {
changed, narrowed, unrepaired, err := w.visit(fd, st, folder)
if errors.Is(err, ErrNoACL) {
return err
}
if err != nil {
unrepaired = true
if w.Failed == "" {
w.Failed = err.Error()
}
}
w.count(folder, changed, narrowed, unrepaired)
return nil
}
// getACL reads one of an entry's ACLs through its descriptor: the access ACL, or a folder's default. An access
// ACL that is not there is the one its mode stands for; a default that is not there is nil.
func getACL(fd int, name string, mode uint32) (acl, error) {
buf := make([]byte, 4+8*32)
for {
n, err := unix.Fgetxattr(fd, name, buf)
switch {
case errors.Is(err, unix.ENODATA):
if name == aclAccessXattr {
return minimalACL(mode), nil
}
return nil, nil
case errors.Is(err, unix.ERANGE):
buf = make([]byte, 2*len(buf))
continue
case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP):
return nil, ErrNoACL
case err != nil:
return nil, fmt.Errorf("an entry's ACL could not be read: %v", err)
}
return decodeACL(buf[:n])
}
}
// setACL writes one of an entry's ACLs through its descriptor, refusing one the kernel would.
func setACL(fd int, name string, a acl) error {
if err := a.check(); err != nil {
return err
}
err := unix.Fsetxattr(fd, name, a.encode(), 0)
switch {
case errors.Is(err, unix.ENOTSUP), errors.Is(err, unix.EOPNOTSUPP):
return ErrNoACL
case err != nil:
return fmt.Errorf("an entry's ACL could not be written: %v", err)
}
return nil
}
// sameACL is whether a and b hold the same entries, whatever their order.
func sameACL(a, b acl) bool {
return string(a.encode()) == string(b.encode())
}
// needs is, for a folder or a file, each kept uid's bits — except its owner's, which its owner's bits give and the
// mesh never changes.
func needs(users map[uint32]bool, owner uint32, folder bool) map[uint32]uint16 {
out := map[uint32]uint16{}
for uid, rw := range users {
if uid != owner {
out[uid] = needOn(rw, folder)
}
}
return out
}
// keepVisit makes users' entries on each entry (ADR 0325 §5): the access ACL with each uid's entry and a mask
// that covers it, and on a folder the same as its default, so what is made under it inherits them. An entry that
// had them is not written. A user that owns the entry is judged by its owner's bits, never given an entry: an
// owner whose bits fall short is unrepaired, said, and never changed.
func keepVisit(users map[uint32]bool) visitFunc {
return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) {
mode := uint32(st.Mode) & 0o7777
need := needs(users, st.Uid, folder)
access, err := getACL(fd, aclAccessXattr, mode)
if err != nil {
return false, false, false, err
}
narrowed, unrepaired := judgeACL(access, users, st, folder)
changed := false
if kept := keptACL(access, need); !sameACL(kept, access) {
if err := setACL(fd, aclAccessXattr, kept); err != nil {
return false, narrowed, true, err
}
changed = true
}
if folder {
def, err := getACL(fd, aclDefaultXattr, mode)
if err != nil {
return changed, narrowed, unrepaired, err
}
base := def
if base == nil {
// No default yet: one whose owner's, owning group's and others' entries are the folder's own, as
// setfacl -d derives it.
base = minimalACL(mode)
if g, ok := access.perm(aclGroupObj, 0); ok {
base[1].Perm = g
}
}
// The default is for what anyone makes under the folder, never its owner alone: every user's entry.
all := map[uint32]uint16{}
for uid, rw := range users {
all[uid] = needOn(rw, true)
}
if kept := keptACL(base, all); def == nil || !sameACL(kept, def) {
if err := setACL(fd, aclDefaultXattr, kept); err != nil {
return changed, narrowed, true, err
}
changed = true
}
}
return changed, narrowed, unrepaired, nil
}
}
// judgeACL is whether an entry with access ACL a gives a user less than its right — by entry and mask together
// (ADR 0325 §6) — and whether that is one the mesh cannot put back: the owner's own bits.
func judgeACL(a acl, users map[uint32]bool, st unix.Stat_t, folder bool) (narrowed, unrepaired bool) {
mode := uint32(st.Mode) & 0o7777
for uid, rw := range users {
need := needOn(rw, folder)
if effective(a, uid, st.Uid, mode)&need == need {
continue
}
narrowed = true
if uid == st.Uid {
unrepaired = true
}
}
return narrowed, unrepaired
}
// dropVisit takes exactly the entries of each uid of drop out of each entry's access ACL and a folder's default
// (ADR 0325 §8): no other entry, and not the mask, so nobody else loses a right.
func dropVisit(drop map[uint32]bool) visitFunc {
return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) {
mode := uint32(st.Mode) & 0o7777
changed := false
for _, name := range []string{aclAccessXattr, aclDefaultXattr} {
if name == aclDefaultXattr && !folder {
continue
}
a, err := getACL(fd, name, mode)
if errors.Is(err, ErrNoACL) {
continue
}
if err != nil {
// Counted unrepaired and said by the walk, which then walks it again: never read as "nothing to drop".
return changed, false, true, err
}
if a == nil {
continue
}
if left := withoutUsers(a, drop); len(left) != len(a) {
if err := setACL(fd, name, left); err != nil {
return changed, false, true, err
}
changed = true
}
}
return changed, false, false, nil
}
}
// dropThenKeep is drop then keep, on one walk: a user whose use changed, or one gone beside one come.
func dropThenKeep(drop map[uint32]bool, users map[uint32]bool) visitFunc {
d, k := dropVisit(drop), keepVisit(users)
return func(fd int, st unix.Stat_t, folder bool) (bool, bool, bool, error) {
c1, _, _, err := d(fd, st, folder)
if err != nil {
return c1, false, false, err
}
c2, n, u, err := k(fd, st, folder)
return c1 || c2, n, u, err
}
}
// folderEmpty is whether the folder open on fd holds nothing yet: the data the mesh may keep without asking.
func folderEmpty(fd int) (bool, error) {
dup, err := unix.Dup(fd)
if err != nil {
return false, err
}
f := os.NewFile(uintptr(dup), "")
defer f.Close()
names, err := f.Readdirnames(1)
switch {
case len(names) > 0:
return false, nil
case err == nil, errors.Is(err, io.EOF):
return true, nil
}
return false, fmt.Errorf("the folder could not be read: %v", err)
}
// keepTop keeps users' entries on the kept folder itself (ADR 0325 §5): what every apply does, beside the walks.
func keepTop(path string, users map[uint32]bool) (changed bool, err error) {
w, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, keepVisit(users))
if err != nil {
return false, err
}
defer unix.Close(fd)
changed, _, _, err = w.visit(fd, st, true)
return changed, err
}
// FolderFacts is what the engine reads of a folder before keeping it: its owner, group and whether it holds data.
type FolderFacts struct {
UID, GID uint32
Empty bool
}
// readFolder reads a folder's facts through the descriptor a mesh directory is opened by.
func readFolder(path string) (FolderFacts, error) {
if err := keptPath(path); err != nil {
return FolderFacts{}, err
}
mw := &meshWalk{want: path}
fd, err := mw.open()
if err != nil {
return FolderFacts{}, err
}
defer unix.Close(fd)
if mw.At != path {
return FolderFacts{}, errors.New("the folder is reached through a link, and the engine keeps no folder " +
"through one")
}
var st unix.Stat_t
if err := unix.Fstat(fd, &st); err != nil {
return FolderFacts{}, err
}
empty, err := folderEmpty(fd)
if err != nil {
return FolderFacts{}, err
}
return FolderFacts{UID: st.Uid, GID: st.Gid, Empty: empty}, nil
}
// CreatePath makes a missing accessed folder (ADR 0325 §9): each missing level made from the nearest existing
// ancestor down, each with that ancestor's owner, group and mode. No link is followed (the ancestor is reached as
// a mesh directory is, and each level is opened with O_NOFOLLOW), no level is made on another filesystem, no file
// is made, and nothing that was there is changed: a level that appears meanwhile stops it. The caller has judged
// that the ancestor is on the placed filesystem and not root's. How many levels it made.
func CreatePath(path string) (int, error) {
return createPath(path, machineStats)
}
func createPath(path string, stats walkStats) (int, error) {
m, err := NearestAncestor(path)
if err != nil {
return 0, fmt.Errorf("the folder cannot be made: %v; nothing was changed", err)
}
if !m.Info.IsDir() {
return 0, errors.New("its nearest existing parent is not a folder (a file or a link), and a folder is " +
"made only below a folder; nothing was changed")
}
mw := &meshWalk{want: m.Ancestor}
fd, err := mw.open()
if err != nil {
return 0, fmt.Errorf("the nearest existing parent is not walked: %v; nothing was changed", err)
}
if mw.At != m.Ancestor {
unix.Close(fd)
return 0, errors.New("the nearest existing parent is reached through a link, and nothing is made " +
"through one; nothing was changed")
}
var anc unix.Stat_t
if err := stats.fd(fd, &anc); err != nil {
unix.Close(fd)
return 0, fmt.Errorf("the nearest existing parent cannot be read: %v; nothing was changed", err)
}
if why := rootOwned(anc.Uid, anc.Gid); why != "" {
unix.Close(fd)
return 0, fmt.Errorf("its nearest existing parent is %s, and nothing is made under root's data "+
"(novox/hq ADR 0325 §12); nothing was changed", why)
}
mode := uint32(anc.Mode) & 0o7777
made := 0
for _, name := range m.Levels {
// Made for its maker alone (the engine, as root), so it is nobody else's until it is the ancestor's.
if err := unix.Mkdirat(fd, name, 0o700); err != nil {
unix.Close(fd)
if errors.Is(err, unix.EEXIST) {
return made, errors.New("a level of the folder appeared while it was made, and nothing that is " +
"there is changed; it stopped there")
}
return made, fmt.Errorf("a level could not be made: %v", err)
}
next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
unix.Close(fd)
if err != nil {
return made, fmt.Errorf("a level made could not be opened: %v", err)
}
fd = next
made++
var st unix.Stat_t
if err := stats.fd(fd, &st); err != nil {
unix.Close(fd)
return made, fmt.Errorf("a level made could not be read: %v", err)
}
if uint64(st.Dev) != uint64(anc.Dev) {
unix.Close(fd)
return made, errors.New("a level made is on another filesystem than its parent, and it stopped there")
}
if st.Uid != anc.Uid || st.Gid != anc.Gid {
if err := unix.Fchown(fd, int(anc.Uid), int(anc.Gid)); err != nil {
unix.Close(fd)
return made, fmt.Errorf("a level made could not be given its parent's owner: %v", err)
}
}
if err := unix.Fchmod(fd, mode); err != nil {
unix.Close(fd)
return made, fmt.Errorf("a level made could not be given its parent's mode: %v", err)
}
}
unix.Close(fd)
return made, nil
}
// MountFsid is the kernel's id of the filesystem at point (statfs f_fsid), in hex; empty when it cannot be read.
// A variable so a test can stand in for a filesystem it cannot mount.
var MountFsid = func(point string) string {
var st unix.Statfs_t
if err := unix.Statfs(point, &st); err != nil {
return ""
}
return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1]))
}
// MountUUID is the UUID of the filesystem at point when it is on a block device: the name in /dev/disk/by-uuid of
// the device the mount's own device number is. Empty for one on no block device (NFS, FUSE, ZFS, a btrfs
// subvolume's own device). A variable so a test can stand in for disks it cannot make.
var MountUUID = func(point string) string {
var st unix.Stat_t
if err := unix.Stat(point, &st); err != nil {
return ""
}
const dir = "/dev/disk/by-uuid"
entries, err := os.ReadDir(dir)
if err != nil {
return ""
}
for _, e := range entries {
var dev unix.Stat_t
if unix.Stat(dir+"/"+e.Name(), &dev) == nil && dev.Mode&unix.S_IFMT == unix.S_IFBLK && dev.Rdev == st.Dev {
return e.Name()
}
}
return ""
}
internal/apply/walk_other.go
+61
View File
@@ -0,0 +1,61 @@
//go:build !linux
package apply
import (
"context"
"errors"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// The node-engine runs on Linux: a folder is kept only by descriptor and through the kernel's ACL attributes,
// which this system has not. Everything refuses, saying so, and changes nothing.
var errNoKeeping = errors.New("a shared folder is kept only by descriptor, which this system cannot; nothing was changed")
// FolderFacts is what the engine reads of a folder before keeping it.
type FolderFacts struct {
UID, GID uint32
Empty bool
}
func readFolder(string) (FolderFacts, error) { return FolderFacts{}, errNoKeeping }
func keepTop(string, map[uint32]bool) (bool, error) { return false, errNoKeeping }
func CreatePath(string) (int, error) { return 0, errNoKeeping }
func ReconcileKept(context.Context) []string { return nil }
func JudgeKept(string, []store.KeptUser, WalkLimits) (WalkCounts, error) {
return WalkCounts{}, errNoKeeping
}
// KeptLook is what one look found of a kept folder.
type KeptLook struct {
Short []string
Busy bool
Sweep store.KeptSweep
Repaired, Unrepaired int
Failed string
}
func LookKept(string, []store.KeptUser, WalkLimits) KeptLook {
return KeptLook{Failed: errNoKeeping.Error()}
}
func TakeFolder(context.Context, *declaration.Declaration, string, string, string, string, Runner) (<-chan struct{},
string, error) {
return nil, "", errNoKeeping
}
// Watcher watches nothing where a folder is not kept.
type Watcher struct{}
func NewWatcher() (*Watcher, error) { return nil, errNoKeeping }
func (*Watcher) Set(map[string][]store.KeptUser) {}
func (*Watcher) Run(context.Context) {}
// MountFsid reads no filesystem id where a folder is not kept.
var MountFsid = func(string) string { return "" }
// MountUUID reads no UUID where a folder is not kept.
var MountUUID = func(string) string { return "" }
internal/apply/watch_linux.go
+350
View File
@@ -0,0 +1,350 @@
//go:build linux
package apply
import (
"context"
"errors"
"strconv"
"sync"
"time"
"unsafe"
"golang.org/x/sys/unix"
"github.com/novox/mesh-host/internal/store"
)
// The watch on each kept folder (novox/hq ADR 0325 §6).
//
// What an identity makes in a kept folder inherits the default entries, and its mask follows the mode its maker
// asked for: a file made at 0644 gives a read-write user only read until its mask is widened. The engine watches
// every folder of each kept tree (inotify: made, moved in, attributes changed) and, on notice, gives the entry its
// users' entries and a mask that covers them — the same keepVisit the walks use, through the entry's own
// descriptor, reached from the kept folder down by O_NOFOLLOW, never through a link and never on another
// filesystem. Notice can be lost (the kernel's queue overflowed) and watches run out (the machine's limit, ENOSPC):
// then the record says so and the sweep finds what was made where no watch is (§10).
// watchMask is what a watch hears: an entry made or moved in, and an entry's mode or ACL changed.
// It is added through /proc/self/fd, whose link is the folder's own descriptor: following it is what reaches the
// folder that was opened, so the watch takes no IN_DONT_FOLLOW.
const watchMask = unix.IN_CREATE | unix.IN_MOVED_TO | unix.IN_ATTRIB | unix.IN_ONLYDIR
// MaxWatchedPerFolder bounds the folders watched under one kept folder: beyond it the sweep alone keeps the rest.
var MaxWatchedPerFolder = 100_000
// Watcher watches every kept folder's tree.
type Watcher struct {
mu sync.Mutex
fd int
dirs map[int]watchedDir
roots map[string]*watchedRoot
}
// watchedDir is one watched folder: its kept folder and the names below it.
type watchedDir struct {
root string
rel []string
}
// watchedRoot is one kept folder watched: its users, its device, and its watches.
type watchedRoot struct {
users map[uint32]bool
dev uint64
wds []int
out bool
}
// NewWatcher opens the machine's notice of changes.
func NewWatcher() (*Watcher, error) {
// Not blocking: Run polls it, so the end of its context ends it whatever the kernel has to say.
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
if err != nil {
return nil, err
}
return &Watcher{fd: fd, dirs: map[int]watchedDir{}, roots: map[string]*watchedRoot{}}, nil
}
// Set is every kept folder to watch, with its users: one newly kept is watched whole (within
// MaxWatchedPerFolder), one no longer kept is let go, and one whose users changed is repaired with the new ones.
func (w *Watcher) Set(folders map[string][]store.KeptUser) {
w.mu.Lock()
defer w.mu.Unlock()
for path, root := range w.roots {
if _, still := folders[path]; !still {
for _, wd := range root.wds {
_, _ = unix.InotifyRmWatch(w.fd, uint32(wd))
delete(w.dirs, wd)
}
delete(w.roots, path)
}
}
for path, users := range folders {
if root, had := w.roots[path]; had {
root.users = usersOf(users)
continue
}
root := &watchedRoot{users: usersOf(users)}
w.roots[path] = root
w.watchTree(path, root)
w.say(path, root)
}
}
// say records how many folders of a kept folder are watched, and whether the watches ran out.
func (w *Watcher) say(path string, root *watchedRoot) {
n, out := len(root.wds), root.out
_ = KeptFolders.Update(path, func(f *store.KeptFolder, had bool) bool {
f.Watches, f.OutOfWatches = n, out
return had
})
}
// watchTree watches a kept folder and every folder under it on its device, by descriptor: each watch is added
// through /proc/self/fd of the folder's own descriptor, so no name is followed to another place.
func (w *Watcher) watchTree(path string, root *watchedRoot) {
_, fd, st, err := openKept(path, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
if err != nil {
return
}
root.dev = uint64(st.Dev)
w.watchBelow(path, root, fd, nil)
}
// watchBelow watches the folder open on fd, rel below root, and every folder under it. It closes fd.
func (w *Watcher) watchBelow(path string, root *watchedRoot, fd int, rel []string) {
defer unix.Close(fd)
if root.out || len(root.wds) >= MaxWatchedPerFolder {
root.out = true
return
}
wd, err := unix.InotifyAddWatch(w.fd, "/proc/self/fd/"+strconv.Itoa(fd), watchMask)
if errors.Is(err, unix.ENOSPC) {
root.out = true
return
}
if err != nil {
return
}
root.wds = append(root.wds, wd)
w.dirs[wd] = watchedDir{root: path, rel: append([]string(nil), rel...)}
dup, err := unix.Dup(fd)
if err != nil {
return
}
names, err := readNames(dup)
if err != nil {
return
}
for _, name := range names {
var st unix.Stat_t
if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR ||
uint64(st.Dev) != root.dev {
continue
}
next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
continue
}
w.watchBelow(path, root, next, append(rel, name))
}
}
// watchPoll is how long Run waits for notice before it looks whether its context ended.
const watchPoll = 250 * time.Millisecond
// Run reads notices until ctx ends, repairing each entry made or changed in a watched folder, and then closes the
// watch: polled, so a quiet machine's watch ends as soon as a busy one's.
func (w *Watcher) Run(ctx context.Context) {
defer unix.Close(w.fd)
buf := make([]byte, 64*1024)
for ctx.Err() == nil {
ready, err := unix.Poll([]unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}}, int(watchPoll/time.Millisecond))
if err != nil && !errors.Is(err, unix.EINTR) {
return
}
if ready <= 0 {
continue
}
n, err := unix.Read(w.fd, buf)
if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR) {
continue
}
if err != nil || n <= 0 {
return
}
for at := 0; at+unix.SizeofInotifyEvent <= n; {
ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[at]))
nameAt := at + unix.SizeofInotifyEvent
name := cString(buf[nameAt : nameAt+int(ev.Len)])
at = nameAt + int(ev.Len)
if ev.Mask&unix.IN_Q_OVERFLOW != 0 {
w.lost()
continue
}
if ev.Mask&unix.IN_IGNORED != 0 {
// The watch is gone — its folder removed, or let go by Set: what pointed at it goes too.
w.forget(int(ev.Wd))
continue
}
w.heard(int(ev.Wd), name, ev.Mask)
}
}
}
// lost is notice lost (the kernel's queue overflowed): every kept folder is watched afresh, and what was made
// meanwhile is the sweep's.
func (w *Watcher) lost() {
w.mu.Lock()
defer w.mu.Unlock()
for path, root := range w.roots {
for _, wd := range root.wds {
_, _ = unix.InotifyRmWatch(w.fd, uint32(wd))
delete(w.dirs, wd)
}
root.wds, root.out = nil, false
w.watchTree(path, root)
w.say(path, root)
}
}
// forget lets go of a watch the kernel removed.
func (w *Watcher) forget(wd int) {
w.mu.Lock()
defer w.mu.Unlock()
dir, ok := w.dirs[wd]
if !ok {
return
}
delete(w.dirs, wd)
if root := w.roots[dir.root]; root != nil {
for i, had := range root.wds {
if had == wd {
root.wds = append(root.wds[:i], root.wds[i+1:]...)
break
}
}
w.say(dir.root, root)
}
}
// watched is how many folders are watched now.
func (w *Watcher) watched() int {
w.mu.Lock()
defer w.mu.Unlock()
return len(w.dirs)
}
// heard repairs one entry a watch heard of: name in the watched folder wd, or the folder itself.
func (w *Watcher) heard(wd int, name string, mask uint32) {
w.mu.Lock()
dir, ok := w.dirs[wd]
root := w.roots[dir.root]
var users map[uint32]bool
var dev uint64
if root != nil {
// Read under the lock: Set gives a root new users while notices are heard.
users, dev = root.users, root.dev
}
w.mu.Unlock()
if !ok || root == nil {
return
}
fd, err := openBelow(dir.root, dir.rel, dev)
if err != nil {
return
}
defer unix.Close(fd)
visit := keepVisit(users)
if name == "" {
var st unix.Stat_t
if unix.Fstat(fd, &st) == nil {
_, _, _, _ = visit(fd, st, true)
}
return
}
var st unix.Stat_t
if unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) != nil || uint64(st.Dev) != dev {
return
}
isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR
if !isDir && st.Mode&unix.S_IFMT != unix.S_IFREG {
return
}
flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK
if isDir {
flags |= unix.O_DIRECTORY
}
efd, err := unix.Openat(fd, name, flags, 0)
if err != nil {
return
}
var opened unix.Stat_t
if unix.Fstat(efd, &opened) != nil || opened.Ino != st.Ino || opened.Dev != st.Dev {
unix.Close(efd)
return
}
_, _, _, _ = visit(efd, opened, isDir)
if isDir && mask&(unix.IN_CREATE|unix.IN_MOVED_TO) != 0 {
// A folder made or moved in is watched, and what was moved in with it is the sweep's.
w.mu.Lock()
defer w.mu.Unlock()
if w.roots[dir.root] != root {
// Let go by Set meanwhile: nothing more is watched for it.
unix.Close(efd)
return
}
w.watchBelow(dir.root, root, efd, append(append([]string(nil), dir.rel...), name))
w.say(dir.root, root)
return
}
unix.Close(efd)
}
// openBelow opens the folder rel below the kept folder root, each level by O_NOFOLLOW on root's device.
func openBelow(root string, rel []string, dev uint64) (int, error) {
_, fd, _, err := openKept(root, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
if err != nil {
return -1, err
}
for _, name := range rel {
next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
unix.Close(fd)
if err != nil {
return -1, err
}
var st unix.Stat_t
if err := unix.Fstat(next, &st); err != nil || uint64(st.Dev) != dev {
unix.Close(next)
return -1, errors.New("a level is on another filesystem")
}
fd = next
}
return fd, nil
}
// readNames reads every name in the folder open on fd, and closes it.
func readNames(fd int) ([]string, error) {
var names []string
buf := make([]byte, 32*1024)
defer unix.Close(fd)
for {
n, err := unix.ReadDirent(fd, buf)
if err != nil {
return names, err
}
if n <= 0 {
return names, nil
}
_, _, names = unix.ParseDirent(buf[:n], -1, names)
}
}
func cString(b []byte) string {
for i, c := range b {
if c == 0 {
return string(b[:i])
}
}
return string(b)
}
internal/declaration/declaration.go
+192 -68
View File
@@ -456,8 +456,10 @@ const (
// **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own —
// it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A
// media library, a download spool is the operator's: it existed before the mesh, several modules
// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The
// host confirms it is there and mounts it; nothing else.
// read and write it at once, and the mesh must not chown, chmod or remove it. The host confirms it
// is there and mounts it, and keeps it usable by the identities that use it through ACL entries it
// adds and nothing else (novox/hq ADR 0325 §5); a missing one is made only below a parent on the
// filesystem the data was placed on (§9).
type Access struct {
ID string `json:"id"`
Type Type `json:"type"`
@@ -465,33 +467,44 @@ type Access struct {
// Mode is how this module reaches the path: read or read-write. Absent narrows to read.
Mode string `json:"mode,omitempty"`
// Grant is who the path's own group is granted to, for a read-write access (novox/hq ADR 0312 §1): the
// identities the module's code runs as. The controller says who; the node-engine reads which group owns
// the path when it applies, because only the machine knows, and never changes the path (§2). Absent is a
// controller older than the grant, or an access that grants nothing — and a read access always is one (§3).
Grant *AccessGrant `json:"grant,omitempty"`
// Users are the identities that use the folder, resolved by the controller (novox/hq ADR 0325 §3, §4): each
// module identity whose container mounts it, and the tool runner's account standing for every module whose
// tools use it. The engine keeps one ACL entry per user on the folder and as its default (§5); several
// accesses naming one folder are kept as one, each uid at the widest use any of them gives it. Absent is a
// controller older than ADR 0325, or an access nobody but others' bits reach (§2): nothing is kept.
Users []AccessUser `json:"users,omitempty"`
// Taken is the controller saying the operator's warrant for this folder was carried out (§8): the folder is
// kept although it holds data. The engine also keeps a folder it took itself, on the act, or found empty.
Taken bool `json:"taken,omitempty"`
// Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted
// so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5).
Grant json.RawMessage `json:"grant,omitempty"`
}
// AccessGrant is the identities a read-write access grants its path's group to (novox/hq ADR 0312 §1).
type AccessGrant struct {
// Containers are the module's containers that mount the path, by id: each is created with the group as a
// supplementary group of its process, and recreated when the group moves (§6).
Containers []string `json:"containers,omitempty"`
// Account is the account that runs the module's tools: put in the group as an account group, as ADR 0252
// §1 puts one, and taken out again as §3 takes back (§5). Empty for a module whose code runs nowhere as
// an account.
Account string `json:"account,omitempty"`
// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4).
const (
FromContainer = "container"
FromToolRunner = "tool-runner"
)
// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its
// use, and where it runs.
type AccessUser struct {
Identity string `json:"identity"`
UID int `json:"uid"`
Use string `json:"use"`
From string `json:"from"`
}
// UnmarshalJSON reads a grant as strictly as a resource is read: a field the engine does not know is a
// thing the controller believes it asked for, and is refused rather than ignored.
func (g *AccessGrant) UnmarshalJSON(raw []byte) error {
type plain AccessGrant
// UnmarshalJSON reads a user as strictly as a resource is read.
func (u *AccessUser) UnmarshalJSON(raw []byte) error {
type plain AccessUser
var p plain
if err := strictDecode(raw, &p); err != nil {
return fmt.Errorf("a grant: %w", err)
return fmt.Errorf("an access's user: %w", err)
}
*g = AccessGrant(p)
*u = AccessUser(p)
return nil
}
@@ -511,27 +524,41 @@ func (a *Access) validate(where string, _ bool) []string {
problems = append(problems, fmt.Sprintf(
"%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode))
}
if a.Grant != nil && a.Mode != AccessReadWrite {
// A read access reads as any account may, and gains no group (novox/hq ADR 0312 §3).
problems = append(problems, where+": a read access grants nothing, and this one names a grant; "+
"only a read-write access grants its path's group (novox/hq ADR 0312 §3)")
}
if a.Grant != nil {
for _, id := range a.Grant.Containers {
if id == "" {
problems = append(problems, where+": a grant names a container with no id")
}
for _, u := range a.Users {
switch {
case u.Identity == "" || !accountName.MatchString(u.Identity):
problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one",
where, u.Identity))
case u.UID <= 0:
// Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a
// number below it is no account.
problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+
"root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID))
}
if a.Grant.Account != "" && !accountName.MatchString(a.Grant.Account) {
// Handed to id, usermod and gpasswd as an argument: a name starting with a dash would be read as an
// option by each of them.
problems = append(problems, fmt.Sprintf("%s: a grant's account is an account's name — letters, digits, "+
"dots, underscores and dashes, not starting with a dash — and %q is not one", where, a.Grant.Account))
if u.Use != AccessRead && u.Use != AccessReadWrite {
problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where,
u.Identity, u.Use, AccessRead, AccessReadWrite))
}
if u.Use == AccessReadWrite && a.Mode != AccessReadWrite {
problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where,
u.Identity, orRead(a.Mode)))
}
if u.From != FromContainer && u.From != FromToolRunner {
problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where,
u.Identity, u.From, FromContainer, FromToolRunner))
}
}
return problems
}
// orRead is an access's mode as said: absent narrows to read.
func orRead(mode string) string {
if mode == "" {
return AccessRead
}
return mode
}
func (u *User) Identity() string { return u.ID }
func (u *User) Kind() Type { return TypeUser }
func (u *User) Target() string { return u.Name }
@@ -1207,6 +1234,78 @@ type Container struct {
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
Health *Health `json:"health,omitempty"`
// RunAs is who the container runs as, resolved by the controller from the module's identity and the
// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
// container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because
// a resource's Identity is its id.
RunAs *ContainerIdentity `json:"identity,omitempty"`
}
// The ways an image is told who to run as (novox/hq ADR 0325 §2).
const (
IdentityByEnv = "env"
IdentityByRun = "run"
)
// ContainerIdentity is who a container runs as, and how its image is told.
type ContainerIdentity struct {
// Mode is IdentityByEnv or IdentityByRun.
Mode string `json:"mode"`
// UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for
// IdentityByEnv: "PUID", "PGID".
UserEnv string `json:"user_env,omitempty"`
GroupEnv string `json:"group_env,omitempty"`
UID int `json:"uid"`
GID int `json:"gid"`
}
// UnmarshalJSON reads an identity as strictly as a resource is read.
func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error {
type plain ContainerIdentity
var p plain
if err := strictDecode(raw, &p); err != nil {
return fmt.Errorf("a container's identity: %w", err)
}
*i = ContainerIdentity(p)
return nil
}
// envName is a variable's name as a runtime takes it.
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
func (i *ContainerIdentity) problems(where string, env map[string]string) []string {
if i == nil {
return nil
}
var problems []string
switch i.Mode {
case IdentityByEnv:
if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) {
problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+
"it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv))
}
for _, name := range []string{i.UserEnv, i.GroupEnv} {
if _, set := env[name]; set && name != "" {
problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+
"for one variable", where, name))
}
}
case IdentityByRun:
if i.UserEnv != "" || i.GroupEnv != "" {
problems = append(problems, where+": an identity told by the runtime's user option names no variable")
}
default:
problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where,
IdentityByEnv, IdentityByRun, i.Mode))
}
// Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started.
if i.UID <= 0 || i.GID <= 0 {
problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+
"never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID))
}
return problems
}
func (c *Container) Identity() string { return c.ID }
@@ -1289,6 +1388,7 @@ func (c *Container) validate(where string, _ bool) []string {
problems = append(problems, where+": networks is for a container that keeps running; a step "+
"runs and exits, and joins nothing afterwards")
}
problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...)
return append(problems, checkImage(where, c.Image)...)
}
@@ -1439,6 +1539,53 @@ type Declaration struct {
// the generation of what it puts back, which may be older than what the machine applied, and is
// not refused for it.
PutBack bool
// Identities is the identity of every module assigned to this machine (novox/hq ADR 0325 §3): an account
// and a group `mesh-<module>` with the numbers the controller allocated once for the module, the same on
// every machine. The engine makes each before any resource is applied, and removes what it made of one no
// longer declared. Absent is a controller older than ADR 0325.
Identities []Identity
}
// Identity is one module's identity, as the controller allocated it.
type Identity struct {
Module string `json:"module"`
Name string `json:"name"`
UID int `json:"uid"`
GID int `json:"gid"`
}
// IdentityPrefix starts every module identity's name.
const IdentityPrefix = "mesh-"
// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number
// twice — two modules with one account would write as each other.
func checkIdentities(ids []Identity) []string {
var problems []string
names, uids, gids := map[string]bool{}, map[int]string{}, map[int]string{}
for _, id := range ids {
where := fmt.Sprintf("identity %q", id.Name)
if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) {
problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s<module>, and this one is "+
"for the module %q", where, IdentityPrefix, id.Module))
}
if id.UID <= 0 || id.GID <= 0 {
problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+
"nor a number below it (novox/hq ADR 0325 §11); nothing of this declaration was made", where,
id.UID, id.GID))
}
if names[id.Name] {
problems = append(problems, where+": declared twice")
}
if other, used := uids[id.UID]; used {
problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other))
}
if other, used := gids[id.GID]; used {
problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, id.GID, other))
}
names[id.Name], uids[id.UID], gids[id.GID] = true, id.Name, id.Name
}
return problems
}
// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
@@ -1608,6 +1755,9 @@ type envelope struct {
// sends them only to a host whose reports say `reads_generation`.
Generation int64 `json:"generation,omitempty"`
PutBack bool `json:"put_back,omitempty"`
// Identities is optional on the wire too (novox/hq ADR 0325 §3). **An older host refuses this key**; a
// controller sends it only to a host whose health says link.IdentityContract.
Identities []Identity `json:"identities,omitempty"`
}
func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1625,8 +1775,9 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
}
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
var problems []string
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack,
Identities: env.Identities}
problems := checkIdentities(env.Identities)
if env.Generation < 0 {
// Below zero would read as "none claimed" and pass every refusal of an older generation.
problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
@@ -1715,7 +1866,6 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
d.Resources = append(d.Resources, resource)
}
problems = append(problems, checkWhileStopped(d.Resources)...)
problems = append(problems, checkGrants(d.Resources)...)
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
problems = append(problems, checkReplaces(d.Resources)...)
if env.Adoption == nil {
@@ -1780,36 +1930,10 @@ func checkWhileStopped(resources []Resource) []string {
return problems
}
// accountName is what a grant's account may be: the portable account name the user tools take, optionally ending
// in `$`, never starting with a dash.
// accountName is what an identity's or an access user's name may be: the portable account name the user tools
// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument.
var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`)
// checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it
// names is a container of this declaration, as a maintenance window's are. A grant to a container the machine
// is not told about would be a group given to nothing, said as granted.
func checkGrants(resources []Resource) []string {
containers := map[string]bool{}
for _, r := range resources {
if r.Kind() == TypeContainer {
containers[r.Identity()] = true
}
}
var problems []string
for _, r := range resources {
a, ok := r.(*Access)
if !ok || a.Grant == nil {
continue
}
for _, id := range a.Grant.Containers {
if id != "" && !containers[id] {
problems = append(problems, fmt.Sprintf("resource %q: its grant names %q, and this declaration "+
"has no container by that id", a.ID, id))
}
}
}
return problems
}
func strictDecode(raw []byte, into any) error {
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
// field the host does not know is a thing the control plane believes it asked for.
internal/declaration/identity_test.go
+61
View File
@@ -0,0 +1,61 @@
package declaration
import (
"strings"
"testing"
)
const anImage = "docker.io/library/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// The fields ADR 0325 adds, read as the controller sends them (novox/hq ADR 0325 §2, §3, §5, issue 567).
func TestIdentitiesUsersAndARunAsAreReadAsSent(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,
"identities":[{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}],
"resources":[
{"id":"radarr.app","type":"container","name":"radarr","image":"` + anImage + `",
"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":61001}},
{"id":"plex.app","type":"container","name":"plex","image":"` + anImage + `","identity":{"mode":"run","uid":61002,"gid":61002}},
{"id":"radarr.movies","type":"access","path":"/srv/media/movies","mode":"read-write","taken":true,
"users":[{"identity":"mesh-radarr","uid":61001,"use":"read-write","from":"container"},
{"identity":"nox","uid":61003,"use":"read","from":"tool-runner"}]},
{"id":"old.movies","type":"access","path":"/srv/media/movies","mode":"read-write","grant":{"account":"nox"}}]}`))
if err != nil {
t.Fatal(err)
}
if len(d.Identities) != 1 || d.Identities[0].UID != 61001 {
t.Fatalf("identities: %+v", d.Identities)
}
if c := d.Resources[0].(*Container); c.RunAs == nil || c.RunAs.UserEnv != "PUID" {
t.Fatalf("a container's identity: %+v", c.RunAs)
}
if a := d.Resources[2].(*Access); !a.Taken || len(a.Users) != 2 || a.Users[1].From != FromToolRunner {
t.Fatalf("an access's users: %+v", a)
}
}
// What the engine refuses on arrival: a root number, an identity not named for its module, a field it does not
// know, a user that writes a read access, a variable set twice.
func TestWhatADeclarationMayNotSayOfAnIdentity(t *testing.T) {
for want, raw := range map[string]string{
"never root": `"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]`,
"named mesh-<module>": `"identities":[{"module":"m","name":"m","uid":61001,"gid":61001}],` +
`"resources":[]`,
"is mesh-a's too": `"identities":[{"module":"a","name":"mesh-a","uid":61001,"gid":61001},` +
`{"module":"b","name":"mesh-b","uid":61001,"gid":61002}],"resources":[]`,
"unknown field": `"identities":[{"module":"m","name":"mesh-m","uid":61001,"gid":61001,"home":"/"}],` +
`"resources":[]`,
"writes an access declared read": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","mode":"read",` +
`"users":[{"identity":"mesh-m","uid":61001,"use":"read-write","from":"container"}]}]`,
"uid 0": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","users":[{"identity":"root","uid":0,` +
`"use":"read","from":"container"}]}]`,
"one value": `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` +
`"env":{"PUID":"1000"},"identity":{"mode":"env","user_env":"PUID","uid":61001,"gid":61001}}]`,
`"env" or "run"`: `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` +
`"identity":{"mode":"sudo","uid":61001,"gid":61001}}]`,
} {
_, err := Parse([]byte(`{"declaration":1,` + raw + `}`))
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("want a refusal saying %q, got %v", want, err)
}
}
}
internal/link/access_act_test.go
+68
View File
@@ -0,0 +1,68 @@
package link
import (
"crypto/ed25519"
"encoding/json"
"strings"
"testing"
"time"
)
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
// signing context of its own. The controller's side holds the same names and words.
func TestTheTakeActsAskKeepsItsFieldNamesAndContext(t *testing.T) {
body, _ := json.Marshal(AccessActAsk{Act: ActTake, Access: "radarr.movies", Warrant: "w"})
if got := keysOf(t, body); got != "access act warrant" {
t.Fatalf("the act's fields are %q", got)
}
if AskAccessActSubject("ace") != "mesh.node.ace.ask.access-act" {
t.Fatalf("the subject is %q", AskAccessActSubject("ace"))
}
if AccessActContext != "novox-mesh access-act v2\n" {
t.Fatalf("the signing context is %q", AccessActContext)
}
}
// signedAct is an act's ask as the controller makes it: the hand-over's fields beside the act's, signed over the
// act's context and its bytes.
func signedAct(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, act AccessActAsk) []byte {
t.Helper()
raw, _ := json.Marshal(struct {
HandOverAsk
AccessActAsk
}{ask, act})
body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(AccessActContext), raw...))})
return body
}
// **Nothing is done without a warrant**, nor for any act but take, nor signed as a hand-over (ADR 0325 §8).
func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
m, key := aMember(t)
now := time.Now()
var given []AccessAct
handle := func(a AccessAct) (string, error) { given = append(given, a); return "started", nil }
check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}}
ask := func(nonce string) HandOverAsk {
return HandOverAsk{Node: m.Node, Path: "/srv/media/movies", By: "the operator", Expires: now.Add(HandOverGood),
Nonce: nonce}
}
for why, body := range map[string][]byte{
"no warrant": signedAct(t, key, ask(aNonce+"01"), AccessActAsk{Act: ActTake, Access: "radarr.movies"}),
"another act": signedAct(t, key, ask(aNonce+"02"), AccessActAsk{Act: "share", Access: "radarr.movies", Warrant: "w"}),
"no access": signedAct(t, key, ask(aNonce+"03"), AccessActAsk{Act: ActTake, Warrant: "w"}),
"a hand-over's": signedAsk(t, key, ask(aNonce+"04")),
"a path as access": signedAct(t, key, ask(aNonce+"05"), AccessActAsk{Act: ActTake, Access: "/srv", Warrant: "w"}),
} {
if a := AnswerAccessAct(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was changed") {
t.Errorf("%s: not refused, or refused without saying nothing was changed: %+v", why, a)
}
}
if len(given) != 0 {
t.Fatalf("a refused ask reached the act: %+v", given)
}
a := AnswerAccessAct(check, handle, signedAct(t, key, ask(aNonce+"06"), AccessActAsk{Act: ActTake,
Access: "radarr.movies", Warrant: "w-1"}))
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
}
}
internal/link/handover.go
+122 -14
View File
@@ -136,20 +136,27 @@ func AnswerHandOver(check HandOverCheck, handle HandOverFunc, body []byte) HandO
// one short line naming who asked, and a path when withPath and none otherwise. The nonce is taken last, after
// every other check passed. It answers the ask with who asked trimmed, or why it is refused.
func believed(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, string) {
ask, _, refused := believedBytes(check, context, withPath, body)
return ask, refused
}
// believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's
// fields: read out of them only once they are believed.
func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) {
if len(body) > MaxHandOverAsk {
return HandOverAsk{}, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body))
return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body))
}
var signed SignedHandOver
if err := json.Unmarshal(body, &signed); err != nil {
return HandOverAsk{}, "the ask is not a signed ask: " + err.Error()
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
}
if len(check.Signer) != ed25519.PublicKeySize ||
!ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) {
return HandOverAsk{}, "the ask is not signed by the mesh"
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
}
var ask HandOverAsk
if err := json.Unmarshal(signed.Ask, &ask); err != nil {
return HandOverAsk{}, "the signed ask is not an ask: " + err.Error()
return HandOverAsk{}, nil, "the signed ask is not an ask: " + err.Error()
}
now := time.Now()
if check.Now != nil {
@@ -163,25 +170,25 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) (
nonce, nonceErr := hex.DecodeString(ask.Nonce)
switch {
case check.Node == "" || ask.Node != check.Node:
return HandOverAsk{}, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node)
return HandOverAsk{}, nil, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node)
case ask.Expires.IsZero() || !now.Before(ask.Expires):
return HandOverAsk{}, "the ask has expired"
return HandOverAsk{}, nil, "the ask has expired"
case ask.Expires.After(now.Add(HandOverGood + HandOverSkew)):
return HandOverAsk{}, "the ask is good for longer than an ask may be"
return HandOverAsk{}, nil, "the ask is good for longer than an ask may be"
case nonceErr != nil || len(nonce) < 16:
return HandOverAsk{}, "the ask carries no nonce of its own"
return HandOverAsk{}, nil, "the ask carries no nonce of its own"
case ask.By == "":
return HandOverAsk{}, "the ask names nobody as asking"
return HandOverAsk{}, nil, "the ask names nobody as asking"
case len(ask.By) > MaxHandOverBy || strings.ContainsAny(ask.By, "\r\n\x00"):
return HandOverAsk{}, "who asked is not one short line"
return HandOverAsk{}, nil, "who asked is not one short line"
case withPath && ask.Path == "":
return HandOverAsk{}, "the ask names no directory"
return HandOverAsk{}, nil, "the ask names no directory"
case !withPath && ask.Path != "":
return HandOverAsk{}, "the ask names a path, and this ask takes none"
return HandOverAsk{}, nil, "the ask names a path, and this ask takes none"
case !nonces.take(ask.Nonce, ask.Expires, now):
return HandOverAsk{}, "the ask was taken already"
return HandOverAsk{}, nil, "the ask was taken already"
}
return ask, ""
return ask, signed.Ask, ""
}
// HandOverReply is what a hand-over request is answered with, and whether to answer: a request with no reply
@@ -320,3 +327,104 @@ func JudgeDirectoriesReply(check HandOverCheck, handle JudgeDirectoriesFunc, rep
type DirectoryJudgements interface {
HearDirectoryJudgements(JudgeDirectoriesFunc)
}
// The take act on an accessed folder, asked over the bus (novox/hq ADR 0325 §8, issue 567).
//
// A folder that already holds data is kept only on the operator's word: the access waits (§7), the controller asks
// the operator once for that folder on that machine (ADR 0259), and on the approved warrant asks this node's engine
// on AskAccessActSubject, signed as a hand-over is and held to the same checks (believed), under a signing context
// of its own, naming the act, the access, its folder as the engine states it, and the warrant. The engine judges
// every value against what it was last declared and what is on disk, answers at once that the act started or why
// it is refused, adds the users' entries to everything under the folder (following no link, entering no other
// filesystem), and states the act on the access's verdict (Act). Adding or dropping a user of a folder already
// taken asks nobody: the engine does it on the apply that declares it.
// AskAccessActSubject is where the controller asks this node's engine to run an act on one access's folder.
func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" }
// AccessActContext is prefixed to an act ask's bytes before signing: never another ask's, nor a declaration's.
const AccessActContext = "novox-mesh access-act v2\n"
// The walks under an accessed folder. Only ActTake is asked; ActAdd and ActDrop are what an apply starts.
const (
// ActTake adds the users' entries to the folder and everything under it, on a warrant (ADR 0325 §8).
ActTake = "take"
// ActAdd adds the entries of a user that joined a taken folder; ActDrop removes exactly a dropped user's.
ActAdd = "add"
ActDrop = "drop"
)
// AccessActAsk is an act's own fields, beside a hand-over's (node, path, by, expires, nonce): which act, the
// access by its resource id, and the warrant the operator approved.
type AccessActAsk struct {
Act string `json:"act"`
Access string `json:"access"`
Warrant string `json:"warrant"`
}
// AccessAct is one act asked and believed, as the handler is given it.
type AccessAct struct {
Act, Access, Path, Warrant, By string
}
// MaxActField bounds an access id and a warrant id.
const MaxActField = 200
// AccessActFunc judges and runs one act: the words to answer with, or the refusal.
type AccessActFunc func(AccessAct) (string, error)
func refusedAct(why string) HandOverAnswer {
return HandOverAnswer{Refused: why + "; nothing was changed"}
}
// AnswerAccessAct is the engine's judgement of one act ask's bytes: believed as a hand-over is, then the act, the
// access and the warrant — **nothing is done without a warrant** — and only then the handler.
func AnswerAccessAct(check HandOverCheck, handle AccessActFunc, body []byte) HandOverAnswer {
if handle == nil {
return refusedAct("this engine runs no act on an accessed folder")
}
ask, signed, refused := believedBytes(check, AccessActContext, true, body)
if refused != "" {
return refusedAct(refused)
}
var act AccessActAsk
if err := json.Unmarshal(signed, &act); err != nil {
return refusedAct("the signed ask is not an act's ask: " + err.Error())
}
act.Access, act.Warrant = strings.TrimSpace(act.Access), strings.TrimSpace(act.Warrant)
switch {
case act.Warrant == "":
return refusedAct("the ask names no warrant, and an act on an accessed folder runs only on one")
case act.Access == "":
return refusedAct("the ask names no access")
case act.Act != ActTake:
return refusedAct(fmt.Sprintf("the ask names the act %q, and the one act asked is %q", act.Act, ActTake))
}
for _, v := range []string{act.Access, act.Warrant} {
if len(v) > MaxActField || strings.ContainsAny(v, "\r\n\x00/") {
return refusedAct("the ask's access or warrant is not one short name")
}
}
said, err := handle(AccessAct{Act: act.Act, Access: act.Access, Path: ask.Path, Warrant: act.Warrant, By: ask.By})
if err != nil {
return refusedAct(err.Error())
}
return HandOverAnswer{Said: said}
}
// AccessActReply is what an act request is answered with, and whether to answer, as HandOverReply.
func AccessActReply(check HandOverCheck, handle AccessActFunc, reply string, body []byte) ([]byte, bool) {
if reply == "" {
return nil, false
}
answer, err := json.Marshal(AnswerAccessAct(check, handle, body))
if err != nil {
return nil, false
}
return answer, true
}
// AccessActs is a link that hears the controller asking this node to run an act on an accessed folder. Optional.
type AccessActs interface {
HearAccessActs(AccessActFunc)
}
internal/link/hearing_nats.go
+24 -1
View File
@@ -66,7 +66,9 @@ type natsLink struct {
searching *nats.Subscription
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
judging *nats.Subscription
node string
// acting hears the controller asking for the take act on an access's folder (novox/hq ADR 0325 §8).
acting *nats.Subscription
node string
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
signer ed25519.PublicKey
arrived chan Declaration
@@ -259,6 +261,24 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
}
}
// HearAccessActs listens for the controller asking this node to run the take act on an access's folder (novox/hq ADR
// 0325 §8), on this machine's own subject, verified as a hand-over is (AnswerAccessAct). A bus whose user list is older
// than the grant refuses the subscription, said by the client; the controller then hears no answer.
func (l *natsLink) HearAccessActs(handle AccessActFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
}
})
if err == nil {
l.acting = acting
}
}
func (l *natsLink) Declarations() <-chan Declaration { return l.arrived }
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
func (l *natsLink) Lost() <-chan error { return l.lost }
@@ -276,6 +296,9 @@ func (l *natsLink) Close() {
if l.searching != nil {
_ = l.searching.Unsubscribe()
}
if l.acting != nil {
_ = l.acting.Unsubscribe()
}
if l.judging != nil {
_ = l.judging.Unsubscribe()
}
internal/link/messages.go
+88 -3
View File
@@ -258,6 +258,14 @@ const RootContract = 3
// know.
const GrantContract = 4
// IdentityContract is the statement of a host that reads a declaration's `identities`, a container's `identity` and
// an access's `users` and `taken` (novox/hq ADR 0325): it makes each module's account, runs a container as its
// module's identity, keeps each shared folder usable through ACL entries, states an access it may not keep yet as
// waiting with a wait of kind access, and hears the take act. It no longer grants an access's group (ADR 0312 is
// superseded): a `grant` still sent is accepted and not acted on. Like the fields before it, its presence is what
// tells the controller this host may be sent those fields; an older host refuses the whole declaration for them.
const IdentityContract = 5
// Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in
// every report, as an event on each change, and again every minute while anything is not healthy — so a
// lost statement is not a lost fault.
@@ -296,11 +304,16 @@ const KindAccount = "account"
// starting ReasonUsedAsFound, until the operator hands it to the mesh (`nox node hand-over <node> <path>` on the control-node, novox/hq issue 356).
const KindDirectory = "directory"
// KindAccess is a read-write access that grants its path's group (novox/hq ADR 0312): healthy while the group may
// write the path, unhealthy while it cannot — the operator's to decide, since the mesh changes nothing on the path
// (§2) — or while the path is owned by a group that is never granted (§4).
// KindAccess is an access whose folder the engine keeps for its users (novox/hq ADR 0325): healthy while every user
// has its effective right on the folder and the sweep's sample under it (§10), waiting while the mesh may not keep
// it yet or its disk is not there (§7), unhealthy otherwise — root's data on its placed filesystem among it (§12).
const KindAccess = "access"
// KindKeptEntries is the mesh's entries on a shared folder that nobody wants and that could not be taken out yet
// (novox/hq ADR 0325 §8): a folder no access names whose disk is away, or entries left on a filesystem no longer at
// a folder. Unhealthy, said under the access that last named it, until the drop walk has run.
const KindKeptEntries = "kept-entries"
// ReasonUsedAsFound starts the reason of a directory used as found.
const ReasonUsedAsFound = "used as found:"
@@ -413,6 +426,78 @@ type ResourceHealth struct {
// Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one
// setting of the module by name, the part, and what the operator gives, in words. Absent otherwise.
Waits []health.Wait `json:"waits,omitempty"`
// AccessWaits is what an access stated waiting waits for (novox/hq ADR 0325 §7). Beside Waits rather than in
// it because mesh-sdk's health.Wait names a secret or a setting and nothing else; it moves there once that type
// has a kind. Absent otherwise.
AccessWaits []AccessWait `json:"access-waits,omitempty"`
// Act is the last walk under an access's folder (ADR 0325 §8): a take on a warrant, or an add or a drop of a
// user's entries. Absent on every other verdict, and on a folder never walked.
Act *AccessActRecord `json:"act,omitempty"`
// Entries is the listing of a kept folder's users, for an access's verdict (ADR 0325 §4): each identity, its
// use, and a note beside the tool runner's entry. Absent on a folder not kept.
Entries []KeptEntry `json:"entries,omitempty"`
}
// WaitKindAccess is the kind of a wait on an access (novox/hq ADR 0325 §7).
const WaitKindAccess = "access"
// AccessWait is one access waiting for the operator: the access by its module's own id for it, the part (its
// module), and what the operator gives, in words with no path (ADR 0234 §6): "let the mesh keep the shared folder
// movies", or "mount the disk that holds books".
type AccessWait struct {
// Kind is WaitKindAccess.
Kind string `json:"kind"`
Access string `json:"access"`
Part string `json:"part"`
What string `json:"what"`
// Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant
// given for another filesystem (the folder moved) does not answer it, and the controller asks again.
Placed string `json:"placed,omitempty"`
}
// KeptEntry is one user of a kept folder as its listing says it.
type KeptEntry struct {
Identity string `json:"identity"`
UID int `json:"uid"`
Use string `json:"use"`
From string `json:"from"`
// Note is said beside the tool runner's entry (ADR 0325 §4): it stands for every module whose tools use the
// folder, at the widest use any of them declares.
Note string `json:"note,omitempty"`
}
// The states of a walk under an accessed folder.
const (
ActRunning = "running"
ActDone = "done"
// ActStopped is a walk that met its bound in entries or time: what it changed stays, and what it did not
// reach is not changed. Asked again, or on the next apply, it goes on.
ActStopped = "stopped"
ActFailed = "failed"
)
// AccessActRecord is one walk under an access's folder, as the engine records it (novox/hq ADR 0325 §8). Counts
// only: never a path.
type AccessActRecord struct {
// Act is ActTake, ActAdd or ActDrop.
Act string `json:"act"`
Access string `json:"access"`
Warrant string `json:"warrant,omitempty"`
By string `json:"by,omitempty"`
State string `json:"state"`
Started time.Time `json:"started"`
Ended time.Time `json:"ended,omitzero"`
// Folders and Files are how many it changed; Kept how many already had their entries; Links the links it
// passed by unfollowed; Other the entries it left that are neither a file nor a folder; Mounts the folders on
// another filesystem it did not enter.
Folders int `json:"folders,omitempty"`
Files int `json:"files,omitempty"`
Kept int `json:"kept,omitempty"`
Links int `json:"links,omitempty"`
Other int `json:"other,omitempty"`
Mounts int `json:"mounts,omitempty"`
// Why is what stopped or failed it, in words with no path.
Why string `json:"why,omitempty"`
}
// SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times
internal/link/queue.go
+5 -2
View File
@@ -62,8 +62,11 @@ type Queue struct {
// JudgeDirectories judges this node's declared directories read-only, asked by the controller over the
// link (novox/hq issue 358). Nil answers every such ask with a refusal.
JudgeDirectories JudgeDirectoriesFunc
Say Announce
Timeout time.Duration
// AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved
// warrant (novox/hq ADR 0325 §8). Nil answers every such ask with a refusal.
AccessAct AccessActFunc
Say Announce
Timeout time.Duration
once sync.Once
wake chan struct{}
internal/link/run.go
+4
View File
@@ -252,6 +252,10 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
if j, ok := link.(DirectoryJudgements); ok {
j.HearDirectoryJudgements(queue.JudgeDirectories)
}
// And asking for the take act on an access's folder, on a warrant (novox/hq ADR 0325 §8).
if s, ok := link.(AccessActs); ok {
s.HearAccessActs(queue.AccessAct)
}
for {
// Asked to stop — by standing aside, say — nothing further is taken in.
internal/store/kept.go
+212
View File
@@ -0,0 +1,212 @@
package store
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
)
// KeptName is the file, beside the node's state, of every shared folder the engine keeps (novox/hq ADR 0325 §5):
// whose entries it added, on which filesystem it was placed, whether it was taken and under which warrant, and how
// far its sweep got. Its own file for the reason the hand-overs have theirs: the take act, the watch and the sweep
// run beside an apply, whose save of the state would lose a record written into it meanwhile.
const KeptName = "kept-folders.json"
// KeptPath is that file beside the state at statePath.
func KeptPath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), KeptName)
}
// KeptUser is one identity the mesh keeps an entry for on a folder: its name, its number, its use and where it
// runs (a container, or the tool runner standing for every module whose tools use the folder, ADR 0325 §4).
type KeptUser struct {
Identity string `json:"identity"`
UID uint32 `json:"uid"`
Use string `json:"use"`
From string `json:"from"`
}
// How a folder came to be kept.
const (
// KeptByWarrant is a folder that held data, taken on the operator's word (ADR 0325 §8).
KeptByWarrant = "warrant"
// KeptEmpty is a folder that held nothing yet when it was first kept (§5).
KeptEmpty = "empty"
// KeptMade is a folder the engine made under a parent on its placed filesystem (§9).
KeptMade = "made"
)
// KeptFolder is one shared folder the engine keeps, or has seen and waits to keep. Counts and mount points only
// beside its own path: never a path under it.
type KeptFolder struct {
Path string `json:"path"`
// Mount is the mount point of the filesystem the folder was placed on when the engine first saw it there:
// later, a nearest existing parent on another one is the disk not there (ADR 0325 §9).
Mount string `json:"mount,omitempty"`
// Kept is that the mesh keeps it; How why (KeptByWarrant, KeptEmpty, KeptMade); Warrant and By the warrant and
// who asked, for a folder taken on one; At since when.
Kept bool `json:"kept,omitempty"`
How string `json:"how,omitempty"`
Warrant string `json:"warrant,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at,omitzero"`
// Users are the identities whose entries are on everything under the folder; Want what the last apply
// declared. Where they differ a walk adds or removes exactly the difference.
Users []KeptUser `json:"users,omitempty"`
Want []KeptUser `json:"want,omitempty"`
// IDs are the accesses that last named the folder: what a leftover is said under once none does.
IDs []string `json:"ids,omitempty"`
// Placed names the filesystem the folder was kept on (the engine's PlacedID: its mount point, source and type),
// so another filesystem mounted at the same point is a move, as another mount point is.
Placed string `json:"placed,omitempty"`
// Fs is what Placed was made from beside the mount point: "uuid:<the filesystem's UUID>", "fsid:<statfs's id>"
// or "source:<type>:<mountinfo's source>". Kept on this machine only, never said. Absent on a record from an
// earlier build, whose Placed is then not compared.
Fs string `json:"fs,omitempty"`
// TakenOn names, as Placed does, the filesystem the operator's warrant was carried out on (ADR 0325 §8): the
// controller's word that the folder was taken holds for that filesystem alone, and a folder moved to another
// asks again. One that names no filesystem this way (a mount point, from an unmerged build) is not taken.
TakenOn string `json:"taken_on,omitempty"`
// Left is the entries the mesh added on a filesystem no longer at the folder (a disk mounted over it, or
// one gone): taken out by a drop walk once that filesystem holds the folder again, said until then.
Left []KeptLeft `json:"left,omitempty"`
// Act is the last walk under the folder: a take, an add or a drop.
Act *KeptAct `json:"act,omitempty"`
// Sweep is how far the sweep got (ADR 0325 §10).
Sweep KeptSweep `json:"sweep"`
// Watches is how many folders under it are watched; OutOfWatches that the machine's watch limit was met, and
// the sweep alone finds what is made where no watch is (§6).
Watches int `json:"watches,omitempty"`
OutOfWatches bool `json:"out_of_watches,omitempty"`
}
// KeptLeft is the mesh's entries left on one filesystem: its mount point and whose entries they are.
type KeptLeft struct {
Mount string `json:"mount"`
// Placed names that filesystem as KeptFolder.Placed does: entries left on the filesystem the folder is on
// now are the folder's own, never walked away.
Placed string `json:"placed,omitempty"`
Fs string `json:"fs,omitempty"`
Users []KeptUser `json:"users"`
// Since is when they were left: a filesystem not back at its mount point within LeftFor is gone, and what was
// left on it is forgotten (said once).
Since time.Time `json:"since,omitzero"`
}
// KeptAct is one walk under a kept folder, as the engine records it. Counts only: never a path.
type KeptAct struct {
// Act is take, add or drop.
Act string `json:"act"`
Warrant string `json:"warrant,omitempty"`
By string `json:"by,omitempty"`
State string `json:"state"`
Started time.Time `json:"started"`
Ended time.Time `json:"ended,omitzero"`
// Folders and Files are the entries changed; Kept the ones that already had what they need; Links the links
// passed by unfollowed; Other the entries neither a file nor a folder; Mounts the folders on another
// filesystem not entered.
Folders int `json:"folders,omitempty"`
Files int `json:"files,omitempty"`
Kept int `json:"kept,omitempty"`
Links int `json:"links,omitempty"`
Other int `json:"other,omitempty"`
Mounts int `json:"mounts,omitempty"`
// Why is what stopped or failed it, in words with no path.
Why string `json:"why,omitempty"`
}
// KeptSweep is the sweep's place in a kept folder: where the last look stopped, below the folder, by the names
// of each level (never said, only resumed from); when it last covered the folder whole; and what this pass found.
type KeptSweep struct {
Cursor []string `json:"cursor,omitempty"`
Covered time.Time `json:"covered,omitzero"`
// Read is how many entries this pass read so far; Narrowed how many gave a user less than its right and were
// put back; Unrepaired how many could not be (an owner's bits, which the mesh never changes, or a write that
// failed).
Read int `json:"read,omitempty"`
Narrowed int `json:"narrowed,omitempty"`
Unrepaired int `json:"unrepaired,omitempty"`
// LastUnrepaired is what the last whole pass could not put back: said until the next pass is through.
LastUnrepaired int `json:"last_unrepaired,omitempty"`
// Bound is that the last look ran out of its bound before the folder was covered.
Bound bool `json:"bound,omitempty"`
}
// KeptFile is the kept folders' file, one writer at a time in this process.
type KeptFile struct {
path string
mu sync.Mutex
}
// NewKeptFile is the kept folders' file beside the state at statePath.
func NewKeptFile(statePath string) *KeptFile { return &KeptFile{path: KeptPath(statePath)} }
// All is every kept or seen folder, by path. None is an empty map; a file that cannot be read is an error.
func (k *KeptFile) All() (map[string]KeptFolder, error) {
k.mu.Lock()
defer k.mu.Unlock()
return k.load()
}
// Update changes the record of one folder under the file's lock: change is given the record (empty when there is
// none) and whether there was one, and says whether to keep it; a record not kept is removed.
func (k *KeptFile) Update(path string, change func(f *KeptFolder, had bool) bool) error {
k.mu.Lock()
defer k.mu.Unlock()
all, err := k.load()
if err != nil {
return err
}
f, had := all[path]
f.Path = path
if change(&f, had) {
all[path] = f
} else {
delete(all, path)
}
return k.save(all)
}
func (k *KeptFile) load() (map[string]KeptFolder, error) {
raw, err := os.ReadFile(k.path)
if errors.Is(err, fs.ErrNotExist) {
return map[string]KeptFolder{}, nil
}
if err != nil {
return nil, fmt.Errorf("reading the kept folders: %w", err)
}
out := map[string]KeptFolder{}
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("the kept folders (%s) are unreadable: %w", k.path, err)
}
return out, nil
}
func (k *KeptFile) save(all map[string]KeptFolder) error {
raw, err := json.MarshalIndent(all, "", " ")
if err != nil {
return err
}
tmp, err := os.CreateTemp(filepath.Dir(k.path), ".kept-folders-*.json")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(append(raw, '\n')); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), k.path)
}
internal/store/store.go
+17
View File
@@ -241,6 +241,23 @@ type State struct {
// network stays declared: undeclaring brings nothing back (ADR 0118), and a private network
// assigned again finds the tunnel's configuration retired rather than missing.
Retired []Retired `json:"retired,omitempty"`
// Identities is every module identity the engine made here (novox/hq ADR 0325 §3): its account and group,
// by name and number, and which of the two it made. An identity no longer declared is removed, and only
// what the engine made: an account or group that was there before is never the mesh's to remove.
Identities []MadeIdentity `json:"identities,omitempty"`
}
// MadeIdentity is one module identity as the engine made it.
type MadeIdentity struct {
Module string `json:"module"`
Name string `json:"name"`
UID int `json:"uid"`
GID int `json:"gid"`
// MadeUser and MadeGroup say which the engine made: only those are removed.
MadeUser bool `json:"made_user,omitempty"`
MadeGroup bool `json:"made_group,omitempty"`
At time.Time `json:"at"`
}
// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq
internal/system/identities.go
+87
View File
@@ -0,0 +1,87 @@
package system
import (
"context"
"fmt"
"strconv"
"strings"
)
// A module's identity as a system account (novox/hq ADR 0325 §3): an account and a group with exactly the numbers
// the controller allocated, no login shell, no home and no password. Optional beside System: a system that has
// none (android) makes no module identity, and the engine says so.
// CreateSystemGroup makes a system group with gid.
func (arch) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error {
if _, err := run(ctx, "groupadd", "--system", "--gid", strconv.Itoa(gid), name); err != nil {
return fmt.Errorf("cannot create the group %q: %w", name, err)
}
return nil
}
// CreateSystemAccount makes a system account with uid and the group gid as its own: no home made, `/` as the
// home field, nologin as its shell, and the password useradd leaves locked.
func (arch) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error {
if _, err := run(ctx, "useradd", "--system", "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(gid),
"--no-create-home", "--home-dir", "/", "--shell", "/usr/bin/nologin", name); err != nil {
return fmt.Errorf("cannot create the account %q: %w", name, err)
}
return nil
}
// RemoveAccount removes an account, leaving every file it owns where it is.
func (arch) RemoveAccount(ctx context.Context, run Runner, name string) error {
if _, err := run(ctx, "userdel", name); err != nil {
return fmt.Errorf("cannot remove the account %q: %w", name, err)
}
return nil
}
// RemoveGroup removes a group.
func (arch) RemoveGroup(ctx context.Context, run Runner, name string) error {
if _, err := run(ctx, "groupdel", name); err != nil {
return fmt.Errorf("cannot remove the group %q: %w", name, err)
}
return nil
}
// CreateSystemGroup makes a system group with gid, with busybox's addgroup.
func (alpine) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error {
if _, err := run(ctx, "addgroup", "-S", "-g", strconv.Itoa(gid), name); err != nil {
return fmt.Errorf("cannot create the group %q: %w", name, err)
}
return nil
}
// CreateSystemAccount makes a system account with busybox's adduser: -S system, -D no password, -H no home made.
func (alpine) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int) error {
group, err := run(ctx, "getent", "group", strconv.Itoa(gid))
if err != nil {
return fmt.Errorf("cannot create the account %q: its group gid %d is not here: %w", name, gid, err)
}
groupName := name
if n, _, ok := strings.Cut(group, ":"); ok {
groupName = n
}
if _, err := run(ctx, "adduser", "-S", "-D", "-H", "-h", "/", "-s", "/sbin/nologin", "-u", strconv.Itoa(uid),
"-G", groupName, name); err != nil {
return fmt.Errorf("cannot create the account %q: %w", name, err)
}
return nil
}
// RemoveAccount removes an account with busybox's deluser.
func (alpine) RemoveAccount(ctx context.Context, run Runner, name string) error {
if _, err := run(ctx, "deluser", name); err != nil {
return fmt.Errorf("cannot remove the account %q: %w", name, err)
}
return nil
}
// RemoveGroup removes a group with busybox's delgroup.
func (alpine) RemoveGroup(ctx context.Context, run Runner, name string) error {
if _, err := run(ctx, "delgroup", name); err != nil {
return fmt.Errorf("cannot remove the group %q: %w", name, err)
}
return nil
}