From f48a3355b3d10e5d4419e8213c53a2396b2a6dfe Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 21:16:05 +0200 Subject: [PATCH] Check every share against where the backup seat keeps its data on every pass, so the hour a clean look is believed cannot hide a new repository --- modules/nfs-server/README.md | 10 +- modules/nfs-server/cmd/nfs-server/backup.go | 172 ++++++++++++++++++ .../nfs-server/cmd/nfs-server/backup_test.go | 116 ++++++++++++ modules/nfs-server/cmd/nfs-server/machine.go | 2 +- modules/nfs-server/cmd/nfs-server/main.go | 17 +- modules/nfs-server/cmd/nfs-server/process.go | 3 + modules/nfs-server/cmd/nfs-server/shares.go | 5 + modules/nfs-server/module.json | 4 + 8 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 modules/nfs-server/cmd/nfs-server/backup.go create mode 100644 modules/nfs-server/cmd/nfs-server/backup_test.go diff --git a/modules/nfs-server/README.md b/modules/nfs-server/README.md index 4630ce55..6f9c9987 100644 --- a/modules/nfs-server/README.md +++ b/modules/nfs-server/README.md @@ -29,7 +29,15 @@ another node requires. - **Never the backups.** A folder that is the machine's root, is not there, is not a folder, is inside a backup (restic) repository (any parent up to `/`), or holds one within four levels below it is refused, said by the tools, and makes the module unhealthy. The look below is bounded (200 000 folders); when it - is cut short, the share is refused rather than exported unchecked. + is cut short, the share is refused rather than exported unchecked. A clean look is believed for an hour. +- **Never where the backup seat keeps its data, checked on every pass.** So that the hour cannot hide a + repository started inside it, the tools ask the controller over the bus, every five minutes, which module + holds `node-backup` on this machine (`seat:mesh-controller.seats`) and every path that module keeps data + in here (`seat:mesh-controller.data`; on the home server the restic repository and its state). They + leave it in `/var/lib/nfs-server/from-bus/backup.json`, and every pass refuses a share at, inside or + holding one of those paths, with nothing kept. A failed ask keeps the last paths and says why. Until a + list has been learnt, what is exported stays exported and the status says the list is not known: a slow + bus is no reason to take the library away, and the look below still guards. - **Only while its filesystem is mounted.** A share that is a filesystem of its own is exported with `mp`, a folder inside one with `mp=`, so an unmounted dataset never exports the empty folder under it. A folder of the root filesystem carries neither. diff --git a/modules/nfs-server/cmd/nfs-server/backup.go b/modules/nfs-server/cmd/nfs-server/backup.go new file mode 100644 index 00000000..b9799bcd --- /dev/null +++ b/modules/nfs-server/cmd/nfs-server/backup.go @@ -0,0 +1,172 @@ +package main + +// Where this machine's backup seat keeps its data, from the bus (follow-up to the review of +// mesh-catalog #136, point 4). +// +// The look below a share for a backup repository walks the folders and is believed for LookKept, so a +// repository that the backup seat starts keeping under a share inside that hour would be exported until the +// look is taken again. So the module also learns, over the bus, which module holds `node-backup` on this +// machine and every path that module keeps data in (its repository among them), and every pass judges +// every share against those paths with nothing kept: a share at, under or above one of them is refused. +// +// The tools ask — the root process does not reach the bus — every BackupEvery, and leave what they learnt +// in BackupFile for the root process. A failed ask keeps the last paths and says why. Until a list has +// been read, what is exported stays exported and the status says the list is not known: the bus being +// slow is no reason to take the library away, and the look below still guards against a repository. +// +// The parsing of mesh-controller.data's answer is a small copy of the mounts module's (DataFrom); shared +// code belongs in mesh-sdk, and moves there when a third module needs it. + +import ( + "encoding/json" + "fmt" + "os" + "path" + "sort" + "time" +) + +// BackupFile is what the tools learnt about the backup seat on this machine. +const BackupFile = WantsDir + "/backup.json" + +// BackupEvery is how often the tools ask. +const BackupEvery = 5 * time.Minute + +// BackupSeat is the seat whose holder keeps this machine's backups. +const BackupSeat = "node-backup" + +// BackupFromBus is the backup seat's holder on this machine and the paths it keeps data in. +type BackupFromBus struct { + Holder string `json:"holder"` + Paths []string `json:"paths"` + Looked string `json:"looked,omitempty"` + // Error is why the last ask got no answer; the holder and paths are then the last answer's. + Error string `json:"error,omitempty"` +} + +// BackupFrom reads the holder of node-backup on one machine from mesh-controller.seats' answer, and the +// paths that module keeps data in on that machine from mesh-controller.data's. A machine whose backup seat +// nobody holds, and an answer with no data list, are errors: never an empty list read as "nothing to fear". +func BackupFrom(machine string, seats, data json.RawMessage) (string, []string, error) { + var s struct { + Answer *struct { + Seats []struct { + Seat string `json:"seat"` + Holders []struct { + Module string `json:"module"` + Node string `json:"node"` + } `json:"holders"` + } `json:"seats"` + } `json:"answer"` + } + if err := json.Unmarshal(seats, &s); err != nil || s.Answer == nil { + return "", nil, fmt.Errorf("the controller's seats answer could not be read: %v", err) + } + holder := "" + for _, seat := range s.Answer.Seats { + if seat.Seat != BackupSeat { + continue + } + for _, h := range seat.Holders { + if h.Node == machine { + holder = h.Module + } + } + } + if holder == "" { + return "", nil, fmt.Errorf("no module holds %s on %s", BackupSeat, machine) + } + var d struct { + Answer *struct { + Data *[]struct { + Machine string `json:"machine"` + Module string `json:"module"` + Path string `json:"path"` + } `json:"data"` + } `json:"answer"` + } + if err := json.Unmarshal(data, &d); err != nil || d.Answer == nil || d.Answer.Data == nil { + return "", nil, fmt.Errorf("the controller's data answer holds no data list (%v)", err) + } + var paths []string + for _, item := range *d.Answer.Data { + if item.Module == holder && (item.Machine == "" || item.Machine == machine) && path.IsAbs(item.Path) { + paths = append(paths, path.Clean(item.Path)) + } + } + sort.Strings(paths) + return holder, paths, nil +} + +// TakeBackup asks the controller over the bus, as the module, through the runtime. +func (m *Machine) TakeBackup(machine string) { + m.TakeBackupWith(machine, askBus) +} + +// TakeBackupWith is TakeBackup with the ask given, for tests. A failed ask keeps the last holder and +// paths and adds why. +func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) (json.RawMessage, error)) { + var last BackupFromBus + if text, err := m.Read(BackupFile); err == nil { + _ = json.Unmarshal([]byte(text), &last) + } + next := last + next.Looked = m.Now().UTC().Format(time.RFC3339) + next.Error = "" + seats, err := ask("seat:mesh-controller.seats", map[string]any{}) + var data json.RawMessage + if err == nil { + data, err = ask("seat:mesh-controller.data", map[string]any{"machine": machine}) + } + if err == nil { + var holder string + var paths []string + holder, paths, err = BackupFrom(machine, seats, data) + if err == nil { + next.Holder, next.Paths = holder, paths + } + } + if err != nil { + next.Error = err.Error() + fmt.Fprintln(os.Stderr, "nfs-server: where the backup seat keeps its data could not be learnt:", err) + } + raw, _ := json.MarshalIndent(next, "", " ") + if werr := m.Write(BackupFile, string(raw)+"\n"); werr != nil { + fmt.Fprintln(os.Stderr, "nfs-server: the backup seat's paths could not be written:", werr) + } +} + +// Backup is what the tools last learnt, and a sentence when nothing usable has been learnt yet. +func (m *Machine) Backup() (BackupFromBus, string) { + text, err := m.Read(BackupFile) + if err != nil { + return BackupFromBus{}, "where this machine's backup seat keeps its repository has not been learnt from the bus yet; the shares are judged by the look below them alone until it is" + } + var b BackupFromBus + if err := json.Unmarshal([]byte(text), &b); err != nil { + return BackupFromBus{}, fmt.Sprintf("what was learnt about the backup seat (%s) could not be read: %v", BackupFile, err) + } + if len(b.Paths) == 0 { + why := "no path has been learnt" + if b.Error != "" { + why = b.Error + } + return b, "where this machine's backup seat keeps its repository is not known (" + why + "); the shares are judged by the look below them alone until it is" + } + return b, "" +} + +// nearBackup is the backup seat's path a real path is at, under or above, or "". +func nearBackup(real string, b BackupFromBus) (string, string) { + for _, p := range b.Paths { + switch { + case real == p: + return p, "is" + case len(real) > len(p) && real[:len(p)] == p && (p == "/" || real[len(p)] == '/'): + return p, "is inside" + case len(p) > len(real) && p[:len(real)] == real && (real == "/" || p[len(real)] == '/'): + return p, "holds" + } + } + return "", "" +} diff --git a/modules/nfs-server/cmd/nfs-server/backup_test.go b/modules/nfs-server/cmd/nfs-server/backup_test.go new file mode 100644 index 00000000..5e576fb4 --- /dev/null +++ b/modules/nfs-server/cmd/nfs-server/backup_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +// Follow-up to the review (point 4): the one-hour look below a share must not hide a backup repository +// that appears later. Where this machine's backup seat keeps its repository is read from the bus and every +// share is judged against it on every pass. + +func (f *fake) backup(holder string, paths ...string) { + raw, _ := json.Marshal(BackupFromBus{Holder: holder, Paths: paths}) + f.file(BackupFile, string(raw)) +} + +func TestAShareHoldingTheBackupSeatsRepositoryIsRefusedOnTheNextPass(t *testing.T) { + f := newFake(t) + f.mountinfo("/") + f.dir("/srv/pool/media") + f.dir("/srv/pool/restic") + f.file(ConfigFile, grantsConfig("pool=/srv/pool media=/srv/pool/media", "pool=ws:rw media=ws:rw")) + f.roster("ws 10.0.0.3") + f.wants(map[string][]Want{"ws": {{Share: "pool", Access: "rw"}, {Share: "media", Access: "rw"}}}) + m := f.machine() + now := time.Date(2026, 10, 8, 20, 0, 0, 0, time.UTC) + m.Now = func() time.Time { return now } + // The repository is not there yet: the look below finds nothing and is kept for an hour. + if s := m.Pass(context.Background(), nil); refusedWhy(s, "pool") != "" { + t.Fatalf("the pool is refused before any repository exists: %+v", s.Refused) + } + // The backup seat starts keeping its repository there; a minute later is well inside LookKept. + f.backup("restic", "/srv/pool/restic", "/var/lib/restic") + now = now.Add(time.Minute) + s := m.Pass(context.Background(), nil) + if why := refusedWhy(s, "pool"); !strings.Contains(why, "/srv/pool/restic") || !strings.Contains(why, "restic") { + t.Errorf("a share holding the backup seat's repository is exported inside the kept look: %q", why) + } + if refusedWhy(s, "media") != "" { + t.Errorf("a share beside the repository is refused: %q", refusedWhy(s, "media")) + } + if strings.Contains(f.read(ExportFile), "/srv/pool ") { + t.Errorf("the export file still names the pool:\n%s", f.read(ExportFile)) + } +} + +func TestAShareInsideTheBackupSeatsRepositoryIsRefused(t *testing.T) { + f := newFake(t) + f.mountinfo("/") + f.dir("/srv/backups/data") + f.file(ConfigFile, grantsConfig("inside=/srv/backups/data", "inside=ws:rw")) + f.roster("ws 10.0.0.3") + f.wants(map[string][]Want{"ws": {{Share: "inside", Access: "rw"}}}) + f.backup("restic", "/srv/backups") + s := f.machine().Pass(context.Background(), nil) + if !strings.Contains(refusedWhy(s, "inside"), "/srv/backups") { + t.Errorf("a share inside the backup seat's repository is not refused: %+v", s.Refused) + } +} + +// Until a list has been read, what is exported stays exported — the bus being slow is no reason to take +// the library away — and the status says the list is not known. +func TestWithoutTheBackupListTheExportsStayAndItIsSaid(t *testing.T) { + f := newFake(t) + f.mountinfo("/") + f.dir("/srv/library") + f.file(ConfigFile, grantsConfig("library=/srv/library", "library=ws:rw")) + f.roster("ws 10.0.0.3") + f.wants(map[string][]Want{"ws": {{Share: "library", Access: "rw"}}}) + s := f.machine().Pass(context.Background(), nil) + if refusedWhy(s, "library") != "" || len(s.Shares) != 1 { + t.Fatalf("the library is not exported while the backup list is unknown: %+v", s) + } + if !strings.Contains(strings.Join(s.Said, "\n"), "backup seat") { + t.Errorf("the status does not say the backup seat's repository is not known: %v", s.Said) + } +} + +func TestBackupFromTheSeatsAndTheData(t *testing.T) { + seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"ace"},{"module":"borg","node":"shanks"}]}]},"ok":true}`) + data := json.RawMessage(`{"answer":{"data":[{"machine":"ace","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"ace","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"ace","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`) + holder, paths, err := BackupFrom("ace", seats, data) + if err != nil || holder != "restic" || strings.Join(paths, " ") != "/storage/restic /var/lib/restic" { + t.Errorf("got %q %v %v", holder, paths, err) + } + if _, _, err := BackupFrom("g14", seats, data); err == nil { + t.Errorf("a machine whose backup seat nobody holds is not said") + } + if _, _, err := BackupFrom("ace", seats, json.RawMessage(`{"ok":true}`)); err == nil { + t.Errorf("an answer with no data list reads as an empty list") + } +} + +// A failed ask keeps the last paths: the file is not emptied, only the error is added. +func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) { + f := newFake(t) + m := f.machine() + m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + return nil, errors.New("no responders") + }) + f.backup("restic", "/storage/restic") + m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + return nil, errors.New("timeout") + }) + var b BackupFromBus + if err := json.Unmarshal([]byte(f.read(BackupFile)), &b); err != nil { + t.Fatal(err) + } + if strings.Join(b.Paths, " ") != "/storage/restic" || !strings.Contains(b.Error, "timeout") { + t.Errorf("a failed ask lost the last paths or did not say why: %+v", b) + } +} diff --git a/modules/nfs-server/cmd/nfs-server/machine.go b/modules/nfs-server/cmd/nfs-server/machine.go index 857a50ed..34476e06 100644 --- a/modules/nfs-server/cmd/nfs-server/machine.go +++ b/modules/nfs-server/cmd/nfs-server/machine.go @@ -232,7 +232,7 @@ func (m *Machine) Glob(pattern string) []string { } // written are the only paths Write may touch. -var written = map[string]bool{ExportFile: true, StatusFile: true, WantsFile: true} +var written = map[string]bool{ExportFile: true, StatusFile: true, WantsFile: true, BackupFile: true} // Write replaces one of the module's own files, atomically. Every other path is refused: the module owns // these three files (the export file and the status, written as root; the wants, written by the tools) diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 12315c53..3406d5a8 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -74,12 +74,25 @@ func ShareExports(m *Machine) { } // The SDK reaches the bus once Serve runs. time.Sleep(2 * time.Second) + m.TakeBackup(machine) put() - for range time.Tick(Every) { - put() + backup := time.Tick(BackupEvery) + every := time.Tick(Every) + for { + select { + case <-backup: + m.TakeBackup(machine) + case <-every: + put() + } } } +// askBus calls a seat's verb over the bus as this module, through the runtime. +func askBus(key string, body any) (json.RawMessage, error) { + return stdio.Ask(key, body) +} + // TakeWants reads every node's `mounts` state from the bus and writes what is asked of this machine to // the wants file, for the root process. When the bus does not answer, the last wants stay. func (m *Machine) TakeWants(machine string) { diff --git a/modules/nfs-server/cmd/nfs-server/process.go b/modules/nfs-server/cmd/nfs-server/process.go index b28b933d..114677ad 100644 --- a/modules/nfs-server/cmd/nfs-server/process.go +++ b/modules/nfs-server/cmd/nfs-server/process.go @@ -53,6 +53,9 @@ func (m *Machine) Pass(ctx context.Context, last *Status) Status { if note != "" { s.Said = append(s.Said, note) } + if _, note := m.Backup(); note != "" { + s.Said = append(s.Said, note) + } exports, refused, said := m.Judge(c, roster, wants) s.Refused = refused s.Said = append(s.Said, said...) diff --git a/modules/nfs-server/cmd/nfs-server/shares.go b/modules/nfs-server/cmd/nfs-server/shares.go index eb656442..3beef7d8 100644 --- a/modules/nfs-server/cmd/nfs-server/shares.go +++ b/modules/nfs-server/cmd/nfs-server/shares.go @@ -338,6 +338,7 @@ func (m *Machine) Judge(c Config, roster map[string]string, wants map[string][]W prefix, _ = netip.ParsePrefix(c.Range) } points := m.MountPoints() + backup, _ := m.Backup() // judged on every pass, never kept: see backup.go shares := map[string]bool{} for _, s := range c.Shares { shares[s.Name] = true @@ -383,6 +384,10 @@ func (m *Machine) Judge(c Config, roster map[string]string, wants map[string][]W no(fmt.Sprintf("The share %s's folder %s%s is owned by root (%d:%d), and every client would act as root there: give it an owner of its own.", s.Name, real, via, o.UID, o.GID)) continue } + if p, how := nearBackup(real, backup); p != "" { + no(fmt.Sprintf("The share %s's folder %s%s %s %s, where this machine's backup seat (%s) keeps its data, and backups are never exported.", s.Name, real, via, how, p, backup.Holder)) + continue + } if m.IsBackupRepository(real) { no(fmt.Sprintf("The share %s's folder %s%s is a backup repository, and backups are never exported.", s.Name, real, via)) continue diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index f36e133c..d67cddde 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -47,6 +47,10 @@ "reads": [ "mounts.shares" ], + "invokes": [ + "seat:mesh-controller.seats", + "seat:mesh-controller.data" + ], "settings": { "grants": { "kind": "preference",