Check every share against where the backup seat keeps its data on every pass, so the hour a clean look is believed cannot hide a new repository
This commit is contained in:
jochen
@@ -29,7 +29,15 @@ another node requires.
|
||||
- **Never the backups.** A folder that is the machine's root, is not there, is not a folder, is inside a
|
||||
backup (restic) repository (any parent up to `/`), or holds one within four levels below it is refused,
|
||||
said by the tools, and makes the module unhealthy. The look below is bounded (200 000 folders); when it
|
||||
is cut short, the share is refused rather than exported unchecked.
|
||||
is cut short, the share is refused rather than exported unchecked. A clean look is believed for an hour.
|
||||
- **Never where the backup seat keeps its data, checked on every pass.** So that the hour cannot hide a
|
||||
repository started inside it, the tools ask the controller over the bus, every five minutes, which module
|
||||
holds `node-backup` on this machine (`seat:mesh-controller.seats`) and every path that module keeps data
|
||||
in here (`seat:mesh-controller.data`; on the home server the restic repository and its state). They
|
||||
leave it in `/var/lib/nfs-server/from-bus/backup.json`, and every pass refuses a share at, inside or
|
||||
holding one of those paths, with nothing kept. A failed ask keeps the last paths and says why. Until a
|
||||
list has been learnt, what is exported stays exported and the status says the list is not known: a slow
|
||||
bus is no reason to take the library away, and the look below still guards.
|
||||
- **Only while its filesystem is mounted.** A share that is a filesystem of its own is exported with `mp`,
|
||||
a folder inside one with `mp=<its mount point>`, so an unmounted dataset never exports the empty folder
|
||||
under it. A folder of the root filesystem carries neither.
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
// Where this machine's backup seat keeps its data, from the bus (follow-up to the review of
|
||||
// mesh-catalog #136, point 4).
|
||||
//
|
||||
// The look below a share for a backup repository walks the folders and is believed for LookKept, so a
|
||||
// repository that the backup seat starts keeping under a share inside that hour would be exported until the
|
||||
// look is taken again. So the module also learns, over the bus, which module holds `node-backup` on this
|
||||
// machine and every path that module keeps data in (its repository among them), and every pass judges
|
||||
// every share against those paths with nothing kept: a share at, under or above one of them is refused.
|
||||
//
|
||||
// The tools ask — the root process does not reach the bus — every BackupEvery, and leave what they learnt
|
||||
// in BackupFile for the root process. A failed ask keeps the last paths and says why. Until a list has
|
||||
// been read, what is exported stays exported and the status says the list is not known: the bus being
|
||||
// slow is no reason to take the library away, and the look below still guards against a repository.
|
||||
//
|
||||
// The parsing of mesh-controller.data's answer is a small copy of the mounts module's (DataFrom); shared
|
||||
// code belongs in mesh-sdk, and moves there when a third module needs it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BackupFile is what the tools learnt about the backup seat on this machine.
|
||||
const BackupFile = WantsDir + "/backup.json"
|
||||
|
||||
// BackupEvery is how often the tools ask.
|
||||
const BackupEvery = 5 * time.Minute
|
||||
|
||||
// BackupSeat is the seat whose holder keeps this machine's backups.
|
||||
const BackupSeat = "node-backup"
|
||||
|
||||
// BackupFromBus is the backup seat's holder on this machine and the paths it keeps data in.
|
||||
type BackupFromBus struct {
|
||||
Holder string `json:"holder"`
|
||||
Paths []string `json:"paths"`
|
||||
Looked string `json:"looked,omitempty"`
|
||||
// Error is why the last ask got no answer; the holder and paths are then the last answer's.
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// BackupFrom reads the holder of node-backup on one machine from mesh-controller.seats' answer, and the
|
||||
// paths that module keeps data in on that machine from mesh-controller.data's. A machine whose backup seat
|
||||
// nobody holds, and an answer with no data list, are errors: never an empty list read as "nothing to fear".
|
||||
func BackupFrom(machine string, seats, data json.RawMessage) (string, []string, error) {
|
||||
var s struct {
|
||||
Answer *struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Holders []struct {
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
} `json:"holders"`
|
||||
} `json:"seats"`
|
||||
} `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(seats, &s); err != nil || s.Answer == nil {
|
||||
return "", nil, fmt.Errorf("the controller's seats answer could not be read: %v", err)
|
||||
}
|
||||
holder := ""
|
||||
for _, seat := range s.Answer.Seats {
|
||||
if seat.Seat != BackupSeat {
|
||||
continue
|
||||
}
|
||||
for _, h := range seat.Holders {
|
||||
if h.Node == machine {
|
||||
holder = h.Module
|
||||
}
|
||||
}
|
||||
}
|
||||
if holder == "" {
|
||||
return "", nil, fmt.Errorf("no module holds %s on %s", BackupSeat, machine)
|
||||
}
|
||||
var d struct {
|
||||
Answer *struct {
|
||||
Data *[]struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Path string `json:"path"`
|
||||
} `json:"data"`
|
||||
} `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &d); err != nil || d.Answer == nil || d.Answer.Data == nil {
|
||||
return "", nil, fmt.Errorf("the controller's data answer holds no data list (%v)", err)
|
||||
}
|
||||
var paths []string
|
||||
for _, item := range *d.Answer.Data {
|
||||
if item.Module == holder && (item.Machine == "" || item.Machine == machine) && path.IsAbs(item.Path) {
|
||||
paths = append(paths, path.Clean(item.Path))
|
||||
}
|
||||
}
|
||||
sort.Strings(paths)
|
||||
return holder, paths, nil
|
||||
}
|
||||
|
||||
// TakeBackup asks the controller over the bus, as the module, through the runtime.
|
||||
func (m *Machine) TakeBackup(machine string) {
|
||||
m.TakeBackupWith(machine, askBus)
|
||||
}
|
||||
|
||||
// TakeBackupWith is TakeBackup with the ask given, for tests. A failed ask keeps the last holder and
|
||||
// paths and adds why.
|
||||
func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) (json.RawMessage, error)) {
|
||||
var last BackupFromBus
|
||||
if text, err := m.Read(BackupFile); err == nil {
|
||||
_ = json.Unmarshal([]byte(text), &last)
|
||||
}
|
||||
next := last
|
||||
next.Looked = m.Now().UTC().Format(time.RFC3339)
|
||||
next.Error = ""
|
||||
seats, err := ask("seat:mesh-controller.seats", map[string]any{})
|
||||
var data json.RawMessage
|
||||
if err == nil {
|
||||
data, err = ask("seat:mesh-controller.data", map[string]any{"machine": machine})
|
||||
}
|
||||
if err == nil {
|
||||
var holder string
|
||||
var paths []string
|
||||
holder, paths, err = BackupFrom(machine, seats, data)
|
||||
if err == nil {
|
||||
next.Holder, next.Paths = holder, paths
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
next.Error = err.Error()
|
||||
fmt.Fprintln(os.Stderr, "nfs-server: where the backup seat keeps its data could not be learnt:", err)
|
||||
}
|
||||
raw, _ := json.MarshalIndent(next, "", " ")
|
||||
if werr := m.Write(BackupFile, string(raw)+"\n"); werr != nil {
|
||||
fmt.Fprintln(os.Stderr, "nfs-server: the backup seat's paths could not be written:", werr)
|
||||
}
|
||||
}
|
||||
|
||||
// Backup is what the tools last learnt, and a sentence when nothing usable has been learnt yet.
|
||||
func (m *Machine) Backup() (BackupFromBus, string) {
|
||||
text, err := m.Read(BackupFile)
|
||||
if err != nil {
|
||||
return BackupFromBus{}, "where this machine's backup seat keeps its repository has not been learnt from the bus yet; the shares are judged by the look below them alone until it is"
|
||||
}
|
||||
var b BackupFromBus
|
||||
if err := json.Unmarshal([]byte(text), &b); err != nil {
|
||||
return BackupFromBus{}, fmt.Sprintf("what was learnt about the backup seat (%s) could not be read: %v", BackupFile, err)
|
||||
}
|
||||
if len(b.Paths) == 0 {
|
||||
why := "no path has been learnt"
|
||||
if b.Error != "" {
|
||||
why = b.Error
|
||||
}
|
||||
return b, "where this machine's backup seat keeps its repository is not known (" + why + "); the shares are judged by the look below them alone until it is"
|
||||
}
|
||||
return b, ""
|
||||
}
|
||||
|
||||
// nearBackup is the backup seat's path a real path is at, under or above, or "".
|
||||
func nearBackup(real string, b BackupFromBus) (string, string) {
|
||||
for _, p := range b.Paths {
|
||||
switch {
|
||||
case real == p:
|
||||
return p, "is"
|
||||
case len(real) > len(p) && real[:len(p)] == p && (p == "/" || real[len(p)] == '/'):
|
||||
return p, "is inside"
|
||||
case len(p) > len(real) && p[:len(real)] == real && (real == "/" || p[len(real)] == '/'):
|
||||
return p, "holds"
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Follow-up to the review (point 4): the one-hour look below a share must not hide a backup repository
|
||||
// that appears later. Where this machine's backup seat keeps its repository is read from the bus and every
|
||||
// share is judged against it on every pass.
|
||||
|
||||
func (f *fake) backup(holder string, paths ...string) {
|
||||
raw, _ := json.Marshal(BackupFromBus{Holder: holder, Paths: paths})
|
||||
f.file(BackupFile, string(raw))
|
||||
}
|
||||
|
||||
func TestAShareHoldingTheBackupSeatsRepositoryIsRefusedOnTheNextPass(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.mountinfo("/")
|
||||
f.dir("/srv/pool/media")
|
||||
f.dir("/srv/pool/restic")
|
||||
f.file(ConfigFile, grantsConfig("pool=/srv/pool media=/srv/pool/media", "pool=ws:rw media=ws:rw"))
|
||||
f.roster("ws 10.0.0.3")
|
||||
f.wants(map[string][]Want{"ws": {{Share: "pool", Access: "rw"}, {Share: "media", Access: "rw"}}})
|
||||
m := f.machine()
|
||||
now := time.Date(2026, 10, 8, 20, 0, 0, 0, time.UTC)
|
||||
m.Now = func() time.Time { return now }
|
||||
// The repository is not there yet: the look below finds nothing and is kept for an hour.
|
||||
if s := m.Pass(context.Background(), nil); refusedWhy(s, "pool") != "" {
|
||||
t.Fatalf("the pool is refused before any repository exists: %+v", s.Refused)
|
||||
}
|
||||
// The backup seat starts keeping its repository there; a minute later is well inside LookKept.
|
||||
f.backup("restic", "/srv/pool/restic", "/var/lib/restic")
|
||||
now = now.Add(time.Minute)
|
||||
s := m.Pass(context.Background(), nil)
|
||||
if why := refusedWhy(s, "pool"); !strings.Contains(why, "/srv/pool/restic") || !strings.Contains(why, "restic") {
|
||||
t.Errorf("a share holding the backup seat's repository is exported inside the kept look: %q", why)
|
||||
}
|
||||
if refusedWhy(s, "media") != "" {
|
||||
t.Errorf("a share beside the repository is refused: %q", refusedWhy(s, "media"))
|
||||
}
|
||||
if strings.Contains(f.read(ExportFile), "/srv/pool ") {
|
||||
t.Errorf("the export file still names the pool:\n%s", f.read(ExportFile))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShareInsideTheBackupSeatsRepositoryIsRefused(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.mountinfo("/")
|
||||
f.dir("/srv/backups/data")
|
||||
f.file(ConfigFile, grantsConfig("inside=/srv/backups/data", "inside=ws:rw"))
|
||||
f.roster("ws 10.0.0.3")
|
||||
f.wants(map[string][]Want{"ws": {{Share: "inside", Access: "rw"}}})
|
||||
f.backup("restic", "/srv/backups")
|
||||
s := f.machine().Pass(context.Background(), nil)
|
||||
if !strings.Contains(refusedWhy(s, "inside"), "/srv/backups") {
|
||||
t.Errorf("a share inside the backup seat's repository is not refused: %+v", s.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
// Until a list has been read, what is exported stays exported — the bus being slow is no reason to take
|
||||
// the library away — and the status says the list is not known.
|
||||
func TestWithoutTheBackupListTheExportsStayAndItIsSaid(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.mountinfo("/")
|
||||
f.dir("/srv/library")
|
||||
f.file(ConfigFile, grantsConfig("library=/srv/library", "library=ws:rw"))
|
||||
f.roster("ws 10.0.0.3")
|
||||
f.wants(map[string][]Want{"ws": {{Share: "library", Access: "rw"}}})
|
||||
s := f.machine().Pass(context.Background(), nil)
|
||||
if refusedWhy(s, "library") != "" || len(s.Shares) != 1 {
|
||||
t.Fatalf("the library is not exported while the backup list is unknown: %+v", s)
|
||||
}
|
||||
if !strings.Contains(strings.Join(s.Said, "\n"), "backup seat") {
|
||||
t.Errorf("the status does not say the backup seat's repository is not known: %v", s.Said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupFromTheSeatsAndTheData(t *testing.T) {
|
||||
seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"ace"},{"module":"borg","node":"shanks"}]}]},"ok":true}`)
|
||||
data := json.RawMessage(`{"answer":{"data":[{"machine":"ace","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"ace","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"ace","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`)
|
||||
holder, paths, err := BackupFrom("ace", seats, data)
|
||||
if err != nil || holder != "restic" || strings.Join(paths, " ") != "/storage/restic /var/lib/restic" {
|
||||
t.Errorf("got %q %v %v", holder, paths, err)
|
||||
}
|
||||
if _, _, err := BackupFrom("g14", seats, data); err == nil {
|
||||
t.Errorf("a machine whose backup seat nobody holds is not said")
|
||||
}
|
||||
if _, _, err := BackupFrom("ace", seats, json.RawMessage(`{"ok":true}`)); err == nil {
|
||||
t.Errorf("an answer with no data list reads as an empty list")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed ask keeps the last paths: the file is not emptied, only the error is added.
|
||||
func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := f.machine()
|
||||
m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("no responders")
|
||||
})
|
||||
f.backup("restic", "/storage/restic")
|
||||
m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("timeout")
|
||||
})
|
||||
var b BackupFromBus
|
||||
if err := json.Unmarshal([]byte(f.read(BackupFile)), &b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(b.Paths, " ") != "/storage/restic" || !strings.Contains(b.Error, "timeout") {
|
||||
t.Errorf("a failed ask lost the last paths or did not say why: %+v", b)
|
||||
}
|
||||
}
|
||||
@@ -232,7 +232,7 @@ func (m *Machine) Glob(pattern string) []string {
|
||||
}
|
||||
|
||||
// written are the only paths Write may touch.
|
||||
var written = map[string]bool{ExportFile: true, StatusFile: true, WantsFile: true}
|
||||
var written = map[string]bool{ExportFile: true, StatusFile: true, WantsFile: true, BackupFile: true}
|
||||
|
||||
// Write replaces one of the module's own files, atomically. Every other path is refused: the module owns
|
||||
// these three files (the export file and the status, written as root; the wants, written by the tools)
|
||||
|
||||
@@ -74,12 +74,25 @@ func ShareExports(m *Machine) {
|
||||
}
|
||||
// The SDK reaches the bus once Serve runs.
|
||||
time.Sleep(2 * time.Second)
|
||||
m.TakeBackup(machine)
|
||||
put()
|
||||
for range time.Tick(Every) {
|
||||
put()
|
||||
backup := time.Tick(BackupEvery)
|
||||
every := time.Tick(Every)
|
||||
for {
|
||||
select {
|
||||
case <-backup:
|
||||
m.TakeBackup(machine)
|
||||
case <-every:
|
||||
put()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askBus calls a seat's verb over the bus as this module, through the runtime.
|
||||
func askBus(key string, body any) (json.RawMessage, error) {
|
||||
return stdio.Ask(key, body)
|
||||
}
|
||||
|
||||
// TakeWants reads every node's `mounts` state from the bus and writes what is asked of this machine to
|
||||
// the wants file, for the root process. When the bus does not answer, the last wants stay.
|
||||
func (m *Machine) TakeWants(machine string) {
|
||||
|
||||
@@ -53,6 +53,9 @@ func (m *Machine) Pass(ctx context.Context, last *Status) Status {
|
||||
if note != "" {
|
||||
s.Said = append(s.Said, note)
|
||||
}
|
||||
if _, note := m.Backup(); note != "" {
|
||||
s.Said = append(s.Said, note)
|
||||
}
|
||||
exports, refused, said := m.Judge(c, roster, wants)
|
||||
s.Refused = refused
|
||||
s.Said = append(s.Said, said...)
|
||||
|
||||
@@ -338,6 +338,7 @@ func (m *Machine) Judge(c Config, roster map[string]string, wants map[string][]W
|
||||
prefix, _ = netip.ParsePrefix(c.Range)
|
||||
}
|
||||
points := m.MountPoints()
|
||||
backup, _ := m.Backup() // judged on every pass, never kept: see backup.go
|
||||
shares := map[string]bool{}
|
||||
for _, s := range c.Shares {
|
||||
shares[s.Name] = true
|
||||
@@ -383,6 +384,10 @@ func (m *Machine) Judge(c Config, roster map[string]string, wants map[string][]W
|
||||
no(fmt.Sprintf("The share %s's folder %s%s is owned by root (%d:%d), and every client would act as root there: give it an owner of its own.", s.Name, real, via, o.UID, o.GID))
|
||||
continue
|
||||
}
|
||||
if p, how := nearBackup(real, backup); p != "" {
|
||||
no(fmt.Sprintf("The share %s's folder %s%s %s %s, where this machine's backup seat (%s) keeps its data, and backups are never exported.", s.Name, real, via, how, p, backup.Holder))
|
||||
continue
|
||||
}
|
||||
if m.IsBackupRepository(real) {
|
||||
no(fmt.Sprintf("The share %s's folder %s%s is a backup repository, and backups are never exported.", s.Name, real, via))
|
||||
continue
|
||||
|
||||
@@ -47,6 +47,10 @@
|
||||
"reads": [
|
||||
"mounts.shares"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.seats",
|
||||
"seat:mesh-controller.data"
|
||||
],
|
||||
"settings": {
|
||||
"grants": {
|
||||
"kind": "preference",
|
||||
|
||||
Reference in New Issue
Block a user