Give every module an identity and keep shared folders through ACL entries (hq ADR 0325, issue 567)
A module's containers ran as whatever their image chose and shared folders were made writable by putting accounts in the folder's group (ADR 0312), which ADR 0325 supersedes. The engine now makes each declared module identity as a system account with the controller's numbers, runs a container declaring `identity` as it, and keeps each placed shared folder usable through one POSIX ACL entry per user, set through the kernel's xattrs on descriptors it opened itself. A folder holding data is kept only after the operator's warrant (the take act); a missing folder is made where its data lives and waits where its disk is not; root's data is never kept. A watch and a bounded sweep put back what an app narrows. The group grant is removed, and what it put an account in is taken back. Reuses the descriptor walks, the create act, the missing-path wait and the warrant- backed act message from the paused issue 567 branch.
This commit is contained in:
@@ -1,181 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// A read-write access whose group cannot write is never quiet (novox/hq ADR 0312 §2).
|
||||
//
|
||||
// The engine grants a read-write access's path group to the identities the module's code runs as, and changes
|
||||
// nothing on the path: whether its group may write is the operator's, as the data is. So an access whose path's
|
||||
// group has no write permission is stated, on every look, as its module's resource of kind `access`, unhealthy,
|
||||
// saying so and that the operator decides — and one owned by a group that is never granted (§4) is stated
|
||||
// unhealthy saying that. The controller raises it as that module's condition on two statements in a row and
|
||||
// clears it on the first healthy one, as it does every resource's health (ADR 0240). The path is the target,
|
||||
// never in the reason: the controller says a reason on the operator's channel, which withholds a path (ADR 0234
|
||||
// §6).
|
||||
|
||||
// grantedAccess is one read-write access that grants, as the judge reads it.
|
||||
type grantedAccess struct {
|
||||
module, id, path string
|
||||
}
|
||||
|
||||
// accessJudge reads, on every look, whether each read-write access that grants may write its path through its
|
||||
// group. Safe for the apply and the looking loop at once.
|
||||
type accessJudge struct {
|
||||
mu sync.Mutex
|
||||
accesses []grantedAccess
|
||||
// was is each access's last verdict, with since when and how many looks in a row found it unhealthy.
|
||||
was map[string]link.ResourceHealth
|
||||
|
||||
// reading guards inFlight: every path whose read has not returned yet.
|
||||
reading sync.Mutex
|
||||
inFlight map[string]bool
|
||||
}
|
||||
|
||||
// accesses is the one access judge on this machine.
|
||||
var accesses = newAccessJudge()
|
||||
|
||||
func newAccessJudge() *accessJudge {
|
||||
return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
|
||||
}
|
||||
|
||||
// set is what the declaration just applied asks: every read-write access of a module that names a grant, not
|
||||
// held as found. An access no longer declared is forgotten.
|
||||
func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) {
|
||||
var as []grantedAccess
|
||||
if d != nil {
|
||||
for _, r := range d.Resources {
|
||||
a, ok := r.(*declaration.Access)
|
||||
if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite || held[a.ID] {
|
||||
continue
|
||||
}
|
||||
at := strings.LastIndex(a.ID, ".")
|
||||
if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
|
||||
continue
|
||||
}
|
||||
as = append(as, grantedAccess{module: a.ID[:at], id: a.ID, path: a.Path})
|
||||
}
|
||||
}
|
||||
sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id })
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.accesses = as
|
||||
declared := map[string]bool{}
|
||||
for _, a := range as {
|
||||
declared[a.id] = true
|
||||
}
|
||||
for id := range j.was {
|
||||
if !declared[id] {
|
||||
delete(j.was, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookBound is how long a look waits for one accessed path to answer. A hard network mount whose server is down
|
||||
// blocks a stat for ever, and one access must never stop every health statement this machine makes.
|
||||
var lookBound = 5 * time.Second
|
||||
|
||||
// pathGroupBounded reads a path's group within lookBound, outside any lock. A read that has not answered is said
|
||||
// so; while it still hangs, later looks do not start another on the same path, so a dead mount costs one blocked
|
||||
// read, not one per look.
|
||||
func (j *accessJudge) pathGroupBounded(path string) (apply.PathGroup, bool, error) {
|
||||
j.reading.Lock()
|
||||
if j.inFlight[path] {
|
||||
j.reading.Unlock()
|
||||
return apply.PathGroup{}, false, nil
|
||||
}
|
||||
j.inFlight[path] = true
|
||||
j.reading.Unlock()
|
||||
type read struct {
|
||||
g apply.PathGroup
|
||||
err error
|
||||
}
|
||||
done := make(chan read, 1)
|
||||
go func() {
|
||||
g, err := apply.GroupOfPath(path)
|
||||
j.reading.Lock()
|
||||
delete(j.inFlight, path)
|
||||
j.reading.Unlock()
|
||||
done <- read{g, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.g, true, r.err
|
||||
case <-time.After(lookBound):
|
||||
return apply.PathGroup{}, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// look reads every access now: healthy when its group may write and may be granted, unhealthy saying why
|
||||
// otherwise. Whether any verdict changed since the last look, for the loop to say it. The paths are read with no
|
||||
// lock held, each within lookBound.
|
||||
func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
|
||||
j.mu.Lock()
|
||||
as := append([]grantedAccess(nil), j.accesses...)
|
||||
j.mu.Unlock()
|
||||
|
||||
type verdict struct{ state, reason string }
|
||||
read := make([]verdict, len(as))
|
||||
for i, a := range as {
|
||||
state, reason := link.StateHealthy, ""
|
||||
g, answered, err := j.pathGroupBounded(a.path)
|
||||
switch {
|
||||
case !answered:
|
||||
state, reason = link.StateUnhealthy, "its path does not answer: reading which group owns it did not "+
|
||||
"return within "+lookBound.String()
|
||||
case err != nil:
|
||||
state, reason = link.StateUnhealthy, "its group cannot be read: "+strings.ReplaceAll(err.Error(), a.path, "the path")
|
||||
case apply.GrantRefusal(g) != nil:
|
||||
state, reason = link.StateUnhealthy, "its group is never granted: "+apply.GrantRefusal(g).Error()
|
||||
case !g.Writable:
|
||||
state, reason = link.StateUnhealthy, apply.CannotWriteWords(g)
|
||||
}
|
||||
read[i] = verdict{state, reason}
|
||||
}
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
changed := false
|
||||
out := make([]link.ResourceHealth, 0, len(as))
|
||||
for i, a := range as {
|
||||
state, reason := read[i].state, read[i].reason
|
||||
v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
|
||||
State: state, Reason: reason, Since: now.UTC()}
|
||||
if was, had := j.was[a.id]; had && was.State == state && was.Reason == reason {
|
||||
v.Since = was.Since
|
||||
v.Streak = was.Streak
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
if state != link.StateHealthy {
|
||||
v.Streak++
|
||||
}
|
||||
j.was[a.id] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// withAccesses adds to a statement every read-write access that grants, as its module's resource of kind access.
|
||||
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
|
||||
h.Resources = append(h.Resources, vs...)
|
||||
return h
|
||||
}
|
||||
|
||||
// accessesHealthy says no access in vs is anything but healthy.
|
||||
func accessesHealthy(vs []link.ResourceHealth) bool {
|
||||
for _, v := range vs {
|
||||
if v.State != link.StateHealthy {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// A read-write access whose path's group cannot write is stated unhealthy on every look, saying so and that the
|
||||
// operator decides, and one whose group is never granted says that (novox/hq ADR 0312 §2, §4). The path is the
|
||||
// target and never in the reason (ADR 0234 §6); a read access, granting nothing, is not stated at all.
|
||||
func TestAnAccessWhoseGroupCannotWriteIsStatedUnhealthyWithTheOperatorsRemedy(t *testing.T) {
|
||||
spool, library, shared, read := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir()
|
||||
groups := map[string]apply.PathGroup{
|
||||
spool: {GID: 1002, Name: "downloads", Writable: false},
|
||||
library: {GID: 1001, Name: "media", Writable: true},
|
||||
shared: {GID: 0, Name: "root", Writable: true},
|
||||
}
|
||||
was := apply.GroupOfPath
|
||||
apply.GroupOfPath = func(path string) (apply.PathGroup, error) { return groups[path], nil }
|
||||
t.Cleanup(func() { apply.GroupOfPath = was })
|
||||
wasRoot := apply.RootGroupIDs
|
||||
apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
|
||||
t.Cleanup(func() { apply.RootGroupIDs = wasRoot })
|
||||
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
|
||||
{"id":"nzbget.downloads","type":"access","path":"` + spool + `","mode":"read-write","grant":{"account":"operator"}},
|
||||
{"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{"account":"operator"}},
|
||||
{"id":"importer.shared","type":"access","path":"` + shared + `","mode":"read-write","grant":{"account":"operator"}},
|
||||
{"id":"player.music","type":"access","path":"` + read + `","mode":"read"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := newAccessJudge()
|
||||
j.set(d, nil)
|
||||
at := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
vs, changed := j.look(at)
|
||||
if !changed || len(vs) != 3 {
|
||||
t.Fatalf("stated %d accesses (changed %v), want the three read-write ones: %+v", len(vs), changed, vs)
|
||||
}
|
||||
got := map[string]link.ResourceHealth{}
|
||||
for _, v := range vs {
|
||||
got[v.Resource] = v
|
||||
if v.Kind != link.KindAccess || strings.Contains(v.Reason, v.Target) {
|
||||
t.Errorf("%s is stated as %q with its path in the reason %q", v.Resource, v.Kind, v.Reason)
|
||||
}
|
||||
}
|
||||
if v := got["nzbget.downloads"]; v.State != link.StateUnhealthy || v.Module != "nzbget" || v.Target != spool ||
|
||||
!strings.Contains(v.Reason, "cannot write") || !strings.Contains(v.Reason, "operator's to decide") || v.Streak != 1 {
|
||||
t.Errorf("an access whose group cannot write is stated %+v", v)
|
||||
}
|
||||
if v := got["importer.shared"]; v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "never granted") {
|
||||
t.Errorf("an access of the root group is stated %+v", v)
|
||||
}
|
||||
if v := got["importer.music"]; v.State != link.StateHealthy {
|
||||
t.Errorf("an access whose group may write is stated %+v", v)
|
||||
}
|
||||
if h := withAccesses(&link.Health{}, vs); accessesHealthy(vs) || len(h.Resources) != 3 {
|
||||
t.Errorf("the statement does not carry the accesses as unhealthy: %+v", h.Resources)
|
||||
}
|
||||
|
||||
// Looked at again: the same verdict, its since kept and its streak counted.
|
||||
vs, changed = j.look(at.Add(time.Minute))
|
||||
for _, v := range vs {
|
||||
if v.Resource == "nzbget.downloads" && (changed || v.Streak != 2 || !v.Since.Equal(at)) {
|
||||
t.Errorf("the second look is %+v (changed %v)", v, changed)
|
||||
}
|
||||
}
|
||||
// The operator gives the group write: healthy at the next look, and nothing the engine did made it so.
|
||||
groups[spool] = apply.PathGroup{GID: 1002, Name: "downloads", Writable: true}
|
||||
vs, _ = j.look(at.Add(2 * time.Minute))
|
||||
for _, v := range vs {
|
||||
if v.Resource == "nzbget.downloads" && v.State != link.StateHealthy {
|
||||
t.Errorf("an access whose group was given write is stated %+v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A path that does not answer — a hard network mount whose server is down — is stated unhealthy within the bound,
|
||||
// and stops neither this look nor the next: one blocked read per path, never one per look.
|
||||
func TestAnAccessWhosePathDoesNotAnswerIsStatedWithinTheBound(t *testing.T) {
|
||||
hung, library := t.TempDir(), t.TempDir()
|
||||
release, finished := make(chan struct{}), make(chan struct{})
|
||||
var started atomic.Int32
|
||||
was, wasBound := apply.GroupOfPath, lookBound
|
||||
apply.GroupOfPath = func(path string) (apply.PathGroup, error) {
|
||||
if path == hung {
|
||||
started.Add(1)
|
||||
<-release
|
||||
defer close(finished)
|
||||
}
|
||||
return apply.PathGroup{GID: 1001, Name: "media", Writable: true}, nil
|
||||
}
|
||||
lookBound = 50 * time.Millisecond
|
||||
wasRoot := apply.RootGroupIDs
|
||||
apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
|
||||
t.Cleanup(func() { apply.GroupOfPath, lookBound, apply.RootGroupIDs = was, wasBound, wasRoot })
|
||||
// Run first: the read that hangs is let go and has returned before anything it read is put back.
|
||||
t.Cleanup(func() { close(release); <-finished })
|
||||
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
|
||||
{"id":"nzbget.downloads","type":"access","path":"` + hung + `","mode":"read-write","grant":{}},
|
||||
{"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := newAccessJudge()
|
||||
j.set(d, nil)
|
||||
for look := 0; look < 3; look++ {
|
||||
done := make(chan []link.ResourceHealth, 1)
|
||||
go func() { vs, _ := j.look(time.Now()); done <- vs }()
|
||||
select {
|
||||
case vs := <-done:
|
||||
for _, v := range vs {
|
||||
switch v.Resource {
|
||||
case "nzbget.downloads":
|
||||
if v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "does not answer") {
|
||||
t.Errorf("look %d: a path that does not answer is stated %+v", look, v)
|
||||
}
|
||||
case "importer.music":
|
||||
if v.State != link.StateHealthy {
|
||||
t.Errorf("look %d: the path that answers is stated %+v", look, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("look %d did not return while one path hangs", look)
|
||||
}
|
||||
}
|
||||
if n := started.Load(); n != 1 {
|
||||
t.Errorf("%d reads were started on the path that hangs; want one", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// An access is judged by what each of its users may do (novox/hq ADR 0325 §7, §10, §12).
|
||||
//
|
||||
// On every look the engine judges each module's access as its module's resource of kind `access`: waiting, with a
|
||||
// wait of kind access, while the mesh may not keep its folder yet or its disk is not there; unhealthy for root's
|
||||
// data on its placed filesystem, and for a kept folder that gives a user less than its right — read by entry and
|
||||
// mask together, on the folder and the sweep's sample under it, which the sweep puts back where it can; healthy
|
||||
// otherwise. The folder is the target, never in a reason nor in a wait's words: the controller says them on the
|
||||
// operator's channel, which withholds a path (ADR 0234 §6).
|
||||
|
||||
// judgedAccess is one access as the judge reads it.
|
||||
type judgedAccess struct {
|
||||
module, id, name, path string
|
||||
taken bool
|
||||
users []store.KeptUser
|
||||
}
|
||||
|
||||
// accessJudge reads every access on every look. Safe for the apply and the looking loop at once.
|
||||
type accessJudge struct {
|
||||
mu sync.Mutex
|
||||
accesses []judgedAccess
|
||||
// was is each access's last verdict, with since when and how many looks in a row found it unhealthy.
|
||||
was map[string]link.ResourceHealth
|
||||
|
||||
// reading guards inFlight: every folder whose look has not returned yet.
|
||||
reading sync.Mutex
|
||||
inFlight map[string]bool
|
||||
}
|
||||
|
||||
// accesses is the one access judge on this machine.
|
||||
var accesses = newAccessJudge()
|
||||
|
||||
func newAccessJudge() *accessJudge {
|
||||
return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
|
||||
}
|
||||
|
||||
// set is what the declaration just applied asks: every access of a module, not held as found, with the users of
|
||||
// its folder as every access of that folder gives them. An access no longer declared is forgotten.
|
||||
func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) {
|
||||
var as []judgedAccess
|
||||
if d != nil {
|
||||
users := map[string]map[uint32]store.KeptUser{}
|
||||
taken := map[string]bool{}
|
||||
for _, r := range d.Resources {
|
||||
if a, ok := r.(*declaration.Access); ok {
|
||||
path := filepath.Clean(a.Path)
|
||||
if users[path] == nil {
|
||||
users[path] = map[uint32]store.KeptUser{}
|
||||
}
|
||||
taken[path] = taken[path] || a.Taken
|
||||
for _, u := range a.Users {
|
||||
had, seen := users[path][uint32(u.UID)]
|
||||
if !seen || (u.Use == declaration.AccessReadWrite && had.Use != declaration.AccessReadWrite) {
|
||||
users[path][uint32(u.UID)] = store.KeptUser{Identity: u.Identity, UID: uint32(u.UID),
|
||||
Use: u.Use, From: u.From}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
a, ok := r.(*declaration.Access)
|
||||
if !ok || held[a.ID] || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
|
||||
continue
|
||||
}
|
||||
at := strings.LastIndex(a.ID, ".")
|
||||
if at <= 0 {
|
||||
continue
|
||||
}
|
||||
path := filepath.Clean(a.Path)
|
||||
ja := judgedAccess{module: a.ID[:at], id: a.ID, name: a.ID[at+1:], path: path, taken: taken[path]}
|
||||
for _, u := range users[path] {
|
||||
ja.users = append(ja.users, u)
|
||||
}
|
||||
sort.Slice(ja.users, func(x, y int) bool { return ja.users[x].UID < ja.users[y].UID })
|
||||
as = append(as, ja)
|
||||
}
|
||||
}
|
||||
sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id })
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.accesses = as
|
||||
declared := map[string]bool{}
|
||||
for _, a := range as {
|
||||
declared[a.id] = true
|
||||
}
|
||||
for id := range j.was {
|
||||
if !declared[id] {
|
||||
delete(j.was, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookBound is how long a look waits for one accessed folder to answer. A hard network mount whose server is down
|
||||
// blocks a stat for ever, and one access must never stop every health statement this machine makes.
|
||||
var lookBound = 5 * time.Second
|
||||
|
||||
// verdict is one access's judgement in a look.
|
||||
type verdict struct {
|
||||
state, reason string
|
||||
waits []link.AccessWait
|
||||
act *link.AccessActRecord
|
||||
entries []link.KeptEntry
|
||||
}
|
||||
|
||||
// judgeBounded judges one access within lookBound, outside any lock. A look that has not answered is said so;
|
||||
// while it still hangs, later looks start no other on the same folder, so a dead mount costs one blocked read.
|
||||
func (j *accessJudge) judgeBounded(a judgedAccess) (verdict, bool) {
|
||||
j.reading.Lock()
|
||||
if j.inFlight[a.path] {
|
||||
j.reading.Unlock()
|
||||
return verdict{}, false
|
||||
}
|
||||
j.inFlight[a.path] = true
|
||||
j.reading.Unlock()
|
||||
done := make(chan verdict, 1)
|
||||
go func() {
|
||||
v := judgeAccess(a)
|
||||
j.reading.Lock()
|
||||
delete(j.inFlight, a.path)
|
||||
j.reading.Unlock()
|
||||
done <- v
|
||||
}()
|
||||
select {
|
||||
case v := <-done:
|
||||
return v, true
|
||||
case <-time.After(lookBound):
|
||||
return verdict{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// judgeAccess is one access's judgement now (ADR 0325 §7, §10, §12), the sweep of its kept folder included.
|
||||
func judgeAccess(a judgedAccess) verdict {
|
||||
v := apply.AccessVerdict(context.Background(), a.path, a.taken, len(a.users) > 0)
|
||||
switch v.Kind {
|
||||
case apply.VerdictWaitKeep:
|
||||
return verdict{state: link.StateWaiting, reason: "it holds data the mesh has not taken, and waits for the " +
|
||||
"operator's word (novox/hq ADR 0325 §7)", waits: []link.AccessWait{{Kind: link.WaitKindAccess,
|
||||
Access: a.name, Part: a.module, What: apply.KeepWords(a.name)}}}
|
||||
case apply.VerdictWaitDisk:
|
||||
return verdict{state: link.StateWaiting, reason: "the disk that holds it is not there (novox/hq ADR 0325 §9)",
|
||||
waits: []link.AccessWait{{Kind: link.WaitKindAccess, Access: a.name, Part: a.module,
|
||||
What: apply.DiskWords(a.name)}}}
|
||||
case apply.VerdictRoot:
|
||||
return verdict{state: link.StateUnhealthy, reason: apply.RootWords(v.Why)}
|
||||
case apply.VerdictBroken:
|
||||
return verdict{state: link.StateUnhealthy, reason: strings.ReplaceAll(v.Why, a.path, "the folder")}
|
||||
case apply.VerdictMake:
|
||||
return verdict{state: link.StateUnhealthy, reason: "it is not there yet; the next apply makes it where " +
|
||||
"its data lives (novox/hq ADR 0325 §9)"}
|
||||
case apply.VerdictNobody:
|
||||
return verdict{state: link.StateHealthy}
|
||||
}
|
||||
out := verdict{state: link.StateHealthy, entries: entriesOf(a.users)}
|
||||
rec, _ := keptNow(a.path)
|
||||
if act := rec.Act; act != nil {
|
||||
out.act = &link.AccessActRecord{Act: act.Act, Access: a.id, Warrant: act.Warrant, By: act.By, State: act.State,
|
||||
Started: act.Started, Ended: act.Ended, Folders: act.Folders, Files: act.Files, Kept: act.Kept,
|
||||
Links: act.Links, Other: act.Other, Mounts: act.Mounts, Why: act.Why}
|
||||
}
|
||||
if !rec.Kept {
|
||||
out.state, out.reason = link.StateStarting, "the controller says it was taken; the next apply keeps it"
|
||||
return out
|
||||
}
|
||||
look := apply.LookKept(a.path, a.users, apply.SweepLimits)
|
||||
var faults []string
|
||||
if look.Failed != "" {
|
||||
faults = append(faults, "the sweep could not look: "+strings.ReplaceAll(look.Failed, a.path, "the folder"))
|
||||
}
|
||||
if len(look.Short) > 0 {
|
||||
faults = append(faults, "the folder itself gives "+strings.Join(look.Short, ", ")+"; put back on the next "+
|
||||
"apply")
|
||||
}
|
||||
if n := look.Sweep.Unrepaired + look.Sweep.LastUnrepaired; n > 0 {
|
||||
faults = append(faults, fmt.Sprintf("%d entries under it give a user less than its right and cannot be "+
|
||||
"put back: an owner's own bits, which the mesh never changes, or an entry the filesystem refused", n))
|
||||
}
|
||||
if len(faults) > 0 {
|
||||
out.state, out.reason = link.StateUnhealthy, strings.Join(faults, "; ")
|
||||
return out
|
||||
}
|
||||
switch {
|
||||
case rec.Act != nil && rec.Act.State == link.ActRunning:
|
||||
out.state, out.reason = link.StateStarting, "the take act is adding its users' entries under it"
|
||||
case !sameUsersList(rec.Users, rec.Want):
|
||||
out.state, out.reason = link.StateStarting, "the entries under it are being changed to its users now"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// entriesOf is a kept folder's listing (ADR 0325 §4): each user, and beside the tool runner's what it stands for.
|
||||
func entriesOf(users []store.KeptUser) []link.KeptEntry {
|
||||
var out []link.KeptEntry
|
||||
for _, u := range users {
|
||||
e := link.KeptEntry{Identity: u.Identity, UID: int(u.UID), Use: u.Use, From: u.From}
|
||||
if u.From == declaration.FromToolRunner {
|
||||
e.Note = "the tool runner's account stands for every module whose tools use this folder, at the widest " +
|
||||
"use any of them declares: a module that declared only read may write through it (novox/hq ADR " +
|
||||
"0325 §4, until issue 569)"
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameUsersList(a, b []store.KeptUser) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := map[uint32]string{}
|
||||
for _, u := range a {
|
||||
seen[u.UID] = u.Use
|
||||
}
|
||||
for _, u := range b {
|
||||
if use, ok := seen[u.UID]; !ok || use != u.Use {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// keptNow is the record of one kept folder, as the engine's file holds it.
|
||||
func keptNow(path string) (store.KeptFolder, bool) {
|
||||
all, err := apply.KeptFolders.All()
|
||||
if err != nil {
|
||||
return store.KeptFolder{}, false
|
||||
}
|
||||
f, had := all[path]
|
||||
return f, had
|
||||
}
|
||||
|
||||
// look reads every access now. Whether any verdict changed since the last look, for the loop to say it.
|
||||
func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
|
||||
j.mu.Lock()
|
||||
as := append([]judgedAccess(nil), j.accesses...)
|
||||
j.mu.Unlock()
|
||||
|
||||
read := make([]verdict, len(as))
|
||||
for i, a := range as {
|
||||
v, answered := j.judgeBounded(a)
|
||||
if !answered {
|
||||
v = verdict{state: link.StateUnhealthy, reason: "its folder does not answer: judging it did not return " +
|
||||
"within " + lookBound.String()}
|
||||
}
|
||||
read[i] = v
|
||||
}
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
changed := false
|
||||
out := make([]link.ResourceHealth, 0, len(as))
|
||||
for i, a := range as {
|
||||
r := read[i]
|
||||
v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
|
||||
State: r.state, Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act, Entries: r.entries}
|
||||
if was, had := j.was[a.id]; had && was.State == r.state && was.Reason == r.reason {
|
||||
v.Since = was.Since
|
||||
v.Streak = was.Streak
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
if r.state == link.StateUnhealthy {
|
||||
v.Streak++
|
||||
}
|
||||
j.was[a.id] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// withAccesses adds to a statement every access, as its module's resource of kind access.
|
||||
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
|
||||
h.Resources = append(h.Resources, vs...)
|
||||
return h
|
||||
}
|
||||
|
||||
// accessesHealthy says no access in vs is anything but healthy.
|
||||
func accessesHealthy(vs []link.ResourceHealth) bool {
|
||||
for _, v := range vs {
|
||||
if v.State != link.StateHealthy {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// keeper runs the walks under kept folders beside the applies, one at a time (ADR 0325 §8): poked after every
|
||||
// apply and every look, it walks whatever folder's entries under it are not what was declared.
|
||||
type keeper struct {
|
||||
running, following atomic.Bool
|
||||
say func(string)
|
||||
watch *apply.Watcher
|
||||
}
|
||||
|
||||
// keeping is the one keeper on this machine.
|
||||
var keeping = &keeper{say: func(string) {}}
|
||||
|
||||
// poke starts a reconcile of the kept folders unless one is running, and has the watch follow what is kept.
|
||||
func (k *keeper) poke(ctx context.Context) {
|
||||
// The watch is set beside, never in the caller's way: watching a new folder's tree reads every folder of it.
|
||||
if k.following.CompareAndSwap(false, true) {
|
||||
go func() {
|
||||
defer k.following.Store(false)
|
||||
k.follow()
|
||||
}()
|
||||
}
|
||||
if !k.running.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer k.running.Store(false)
|
||||
for _, line := range apply.ReconcileKept(ctx) {
|
||||
k.say(line)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// follow has the watch watch every kept folder with its users.
|
||||
func (k *keeper) follow() {
|
||||
if k.watch == nil {
|
||||
return
|
||||
}
|
||||
all, err := apply.KeptFolders.All()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
folders := map[string][]store.KeptUser{}
|
||||
for path, f := range all {
|
||||
if f.Kept && len(f.Want) > 0 {
|
||||
folders[path] = f.Want
|
||||
}
|
||||
}
|
||||
k.watch.Set(folders)
|
||||
}
|
||||
|
||||
// startKeeping opens the watch and has the keeper say what it does. A machine without the watch is said so, and
|
||||
// its sweep alone finds what is made.
|
||||
func startKeeping(ctx context.Context, say func(string)) {
|
||||
keeping.say = say
|
||||
w, err := apply.NewWatcher()
|
||||
if err != nil {
|
||||
say(fmt.Sprintf("the kept folders are not watched (%v): the sweep alone finds what is made in them", err))
|
||||
} else {
|
||||
keeping.watch = w
|
||||
go w.Run(ctx)
|
||||
}
|
||||
keeping.poke(ctx)
|
||||
}
|
||||
|
||||
// accessActFrom is the take act as the controller asks it over the link (ADR 0325 §8): judged against the
|
||||
// declaration this node keeps, and run beside the applies.
|
||||
func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
|
||||
return func(act link.AccessAct) (string, error) {
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return "", errors.New("this node keeps no declaration, so it is declared no access")
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
d, err := declaration.Parse(kept.Declaration)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
_, said, err := apply.TakeFolder(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By,
|
||||
apply.ExecRunner)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
|
||||
return "", err
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
}
|
||||
|
||||
// withWaiting is held with every id of waiting beside it: what liveness does not judge.
|
||||
func withWaiting(held, waiting map[string]bool) map[string]bool {
|
||||
if len(waiting) == 0 {
|
||||
return held
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for id := range held {
|
||||
out[id] = true
|
||||
}
|
||||
for id := range waiting {
|
||||
out[id] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Row 4: the listing of a kept folder says, beside the tool runner's entry, that it stands for every module whose
|
||||
// tools use the folder at the widest use (novox/hq ADR 0325 §4).
|
||||
func TestTheListingSaysWhatTheToolRunnersEntryStandsFor(t *testing.T) {
|
||||
entries := entriesOf([]store.KeptUser{
|
||||
{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: declaration.FromContainer},
|
||||
{Identity: "nox", UID: 61003, Use: "read-write", From: declaration.FromToolRunner},
|
||||
})
|
||||
if entries[0].Note != "" || !strings.Contains(entries[1].Note, "widest use") {
|
||||
t.Fatalf("the listing: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// Row 7, the engine's part: a folder that holds data and was never taken is said waiting with one wait of kind
|
||||
// access — the access by its module's own id, the part its module, and words with no path (novox/hq ADR 0325 §7).
|
||||
func TestAnAccessTheMeshMayNotKeepYetIsSaidAsAWaitOfItsOwnKind(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
lib := filepath.Join(dir, "movies")
|
||||
if err := os.MkdirAll(filepath.Join(lib, "a film"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := judgeAccess(judgedAccess{module: "radarr", id: "radarr.movies", name: "movies", path: lib,
|
||||
users: []store.KeptUser{{Identity: "mesh-radarr", UID: 61001, Use: "read-write", From: "container"}}})
|
||||
if v.state != link.StateWaiting || len(v.waits) != 1 {
|
||||
t.Fatalf("not waiting with one wait: %+v", v)
|
||||
}
|
||||
raw, _ := json.Marshal(v.waits[0])
|
||||
if string(raw) != `{"kind":"access","access":"movies","part":"radarr","what":"let the mesh keep the shared folder movies"}` {
|
||||
t.Fatalf("the wait reads %s", raw)
|
||||
}
|
||||
if strings.Contains(v.reason, dir) {
|
||||
t.Fatalf("the reason names the path: %s", v.reason)
|
||||
}
|
||||
}
|
||||
+23
-12
@@ -241,6 +241,8 @@ func run(ctx context.Context, command string, opts options) error {
|
||||
// found (novox/hq issue 339).
|
||||
handedOverFrom(opts.state)
|
||||
foundState.Store(opts.state)
|
||||
// And what it knows of the shared folders it keeps is beside its state (novox/hq ADR 0325 §5).
|
||||
apply.KeptFolders = store.NewKeptFile(opts.state)
|
||||
// And the guard asks the running container runtimes where they keep their data (novox/hq issue 339).
|
||||
apply.AskRuntimes = apply.ExecRunner
|
||||
switch command {
|
||||
@@ -1121,6 +1123,8 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224).
|
||||
sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
go sched.Run(ctx)
|
||||
// The watch on every kept folder and the walks under them, beside the applies (novox/hq ADR 0325 §6, §8).
|
||||
startKeeping(ctx, func(line string) { say(line) })
|
||||
|
||||
// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
|
||||
// report this host makes carries what it says.
|
||||
@@ -1232,8 +1236,10 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// A read-only judgement of the directories this node is declared, asked by the controller (novox/hq
|
||||
// issue 358): which the walk that follows no link another account could make would refuse.
|
||||
JudgeDirectories: judgeDirectoriesFrom(opts.state),
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
// The take act on an access's folder, on the operator's warrant (novox/hq ADR 0325 §8).
|
||||
AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
}
|
||||
worker := make(chan struct{})
|
||||
go func() {
|
||||
@@ -1596,7 +1602,8 @@ const (
|
||||
// judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
|
||||
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
||||
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
||||
// rule 6).
|
||||
// rule 6) — but for the sweep of a kept folder, which ADR 0325 §10 has every look make: it puts back an entry or a
|
||||
// mask an app narrowed, and nothing else.
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
|
||||
ticker := time.NewTicker(liveness.LookEvery)
|
||||
defer ticker.Stop()
|
||||
@@ -1635,8 +1642,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every read-write access that grants, read on every look (novox/hq ADR 0312 §2).
|
||||
// Every access, judged on every look, its kept folder swept (novox/hq ADR 0325 §10); and a walk stopped
|
||||
// at its bound goes on.
|
||||
accessSt, accessesChanged := accesses.look(time.Now())
|
||||
keeping.poke(ctx)
|
||||
owed = owed || accessesChanged
|
||||
if accessesChanged {
|
||||
for _, v := range accessSt {
|
||||
@@ -1691,10 +1700,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
|
||||
// healthAsReported is a statement as the report and the event carry it.
|
||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||
// GrantContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
// user's declared `root` (novox/hq ADR 0266), and grants a read-write access's path group (novox/hq ADR 0312),
|
||||
// which is what tells the controller it may be sent any of those fields.
|
||||
h := &link.Health{Contract: link.GrantContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
// IdentityContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, a container's identity and
|
||||
// an access's users (novox/hq ADR 0325), which is what tells the controller it may be sent any of those fields.
|
||||
h := &link.Health{Contract: link.IdentityContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
for _, r := range st.Resources {
|
||||
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
|
||||
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
|
||||
@@ -2075,7 +2084,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
|
||||
// applied, so every resource this apply started says `starting` and the gate waits for it.
|
||||
if j := judging; j != nil {
|
||||
j.Set(liveness.LongRunning(declared, held))
|
||||
// A container that waits for an accessed folder not there was not created: the access says why, and
|
||||
// liveness judges it neither alive nor dead (novox/hq ADR 0325 §9).
|
||||
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnAccess(declared))))
|
||||
st, _ := j.Look(ctx)
|
||||
netJudge.declare(declaredResolvConf(declared))
|
||||
var netSt *network.Statement
|
||||
@@ -2093,13 +2104,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
// said with it, relogin needed included, in the report that says the apply.
|
||||
var accountSt *accounts.Statement
|
||||
if a := accountJudge; a != nil {
|
||||
// A read-write access's account too (novox/hq ADR 0312 §6): relogin needed is said for a session of
|
||||
// the account's own manager that began before the grant, as for a user resource's group.
|
||||
a.Set(append(accounts.Of(declared, held), apply.GrantedAccounts(declared, held)...))
|
||||
a.Set(accounts.Of(declared, held))
|
||||
as, _ := a.Look(ctx)
|
||||
accountSt = &as
|
||||
}
|
||||
accesses.set(declared, held)
|
||||
// What the apply changed in who uses a kept folder is walked under it now, beside (ADR 0325 §8).
|
||||
keeping.poke(ctx)
|
||||
accessSt, _ := accesses.look(time.Now())
|
||||
report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
|
||||
accountSt), foundDirectories(updated)), accessSt)
|
||||
|
||||
@@ -87,8 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
|
||||
}}
|
||||
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||
// At RootContract or later: an engine that also grants an access's group says GrantContract (novox/hq ADR 0312).
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.GrantContract {
|
||||
// At RootContract or later: an engine that also reads identities and keeps shared folders says IdentityContract
|
||||
// (novox/hq ADR 0325), one above ADR 0312's GrantContract.
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.IdentityContract ||
|
||||
link.IdentityContract != link.GrantContract+1 {
|
||||
t.Fatalf("contract %d", h.Contract)
|
||||
}
|
||||
got := map[string]string{}
|
||||
|
||||
Reference in New Issue
Block a user