package main import ( "context" "encoding/json" "fmt" "net/netip" "os" "path" "strings" "time" ) // The module's process, as root: every Every it reads the assignment's shares, renders the export file, // has the kernel read it again when anything changed or a share is not live, and writes what it saw for // the tools (novox/hq ADR 0263). // Every is how often the process looks. const Every = 30 * time.Second // Status is the process's last look, as the tools read it. type Status struct { At time.Time `json:"at"` Range string `json:"range"` Shares []Exported `json:"shares"` Refused []Refused `json:"refused,omitempty"` Problems []string `json:"problems,omitempty"` // Said is what does not match between the grants and the wants: a node wanting a share it is not // granted, a grant nobody wants, a share exported to no node. Said []string `json:"said,omitempty"` // Warnings make the module no less healthy but are said in its health: what it could not learn, and // how old what it relies on is. Warnings []string `json:"warnings,omitempty"` // Reloaded is when the process last had the kernel read the export files again, and why. Reloaded *time.Time `json:"reloaded,omitempty"` ReloadWhy string `json:"reload_why,omitempty"` ReloadError string `json:"reload_error,omitempty"` } // Pass is one look: what it did is in the status it answers, which it has written. func (m *Machine) Pass(ctx context.Context, last *Status) Status { s := Status{At: m.Now().UTC()} if last != nil { s.Reloaded, s.ReloadWhy = last.Reloaded, last.ReloadWhy } text, err := m.Read(ConfigFile) if err != nil { s.Problems = append(s.Problems, fmt.Sprintf("the assignment's shares (%s) could not be read: %v; nothing is exported", ConfigFile, err)) } c, problems := ParseConfig(text) s.Problems = append(s.Problems, problems...) s.Range = c.Range roster, note := m.Roster() if note != "" { s.Problems = append(s.Problems, note) } wants, note := m.Wants() if note != "" { s.Said = append(s.Said, note) } if _, note := m.Backup(); note != "" { s.Warnings = append(s.Warnings, note) } m.setWalk(ctx) exports, refused, said := m.Judge(c, roster, wants) m.setWalk(nil) if ctx.Err() != nil { // Stopping: write nothing and reload nothing, so Stopped starts at once. s.Problems = append(s.Problems, "the pass was ended by the process stopping; nothing was written") return s } s.Refused = refused s.Said = append(s.Said, said...) want := ExportFileText(exports) have, _ := m.Read(ExportFile) reload := "" if have != want { if err := m.Write(ExportFile, want); err != nil { s.Problems = append(s.Problems, fmt.Sprintf("the export file could not be written: %v", err)) } else { reload = "the export file changed" } } live, note := m.Live() foreign, _, _ := m.Foreign() mine := ParseExports(want, ExportFile) if reload == "" && last == nil { reload = "the process started, and the kernel's table is made to match the file" } if reload == "" && last != nil && last.ReloadError != "" { reload = "the last reload failed: " + last.ReloadError } if reload == "" { for _, e := range exports { if !SharedLive(live, e) { reload = fmt.Sprintf("the share %s was not in the kernel's table", e.Name) break } } } if reload == "" { if stray := Stray(live, mine, foreign, c.Range); len(stray) > 0 { reload = fmt.Sprintf("the kernel exports %s to %s, which the export file no longer carries", stray[0].Path, stray[0].Client) } } if reload != "" { if _, err := m.AsRootWithin(ctx, "exportfs", "-ra"); err != nil { s.ReloadError = err.Error() } else { at := m.Now().UTC() s.Reloaded, s.ReloadWhy = &at, reload } live, note = m.Live() } if note != "" && len(exports) > 0 { s.Problems = append(s.Problems, note) } for _, e := range Stray(live, mine, foreign, c.Range) { s.Problems = append(s.Problems, fmt.Sprintf("the kernel still exports %s to %s, which the module's export file "+ "no longer carries: a reload has to take it away (exportfs -ra as root)", e.Path, e.Client)) } for i := range exports { exports[i].Live = SharedLive(live, exports[i]) } s.Shares = exports raw, _ := json.MarshalIndent(s, "", " ") if err := m.Write(StatusFile, string(raw)+"\n"); err != nil { s.Problems = append(s.Problems, fmt.Sprintf("the status file could not be written: %v", err)) } return s } // ProcessUnit is the service the node-engine runs this process as: the manifest's process name, `.service`. const ProcessUnit = "nfs-server-exports.service" // StopTimeout bounds Stopped and the commands it runs: well under the service manager's stop timeout (90 s, // then SIGKILL), so the export file is gone before the process could be killed (mesh-host #62). var StopTimeout = 10 * time.Second // RunExports is the process: a pass now and every Every until the context ends, then Stopped. func (m *Machine) RunExports(ctx context.Context) error { last := m.Pass(ctx, nil) t := time.NewTicker(Every) defer t.Stop() for { select { case <-ctx.Done(): m.Stopped() return nil case <-t.C: last = m.Pass(ctx, &last) } } } // LastStatus is the process's last look, or why there is none. func (m *Machine) LastStatus() (*Status, error) { text, err := m.Read(StatusFile) if err != nil { return nil, fmt.Errorf("the module's process has written no status (%s): %v", StatusFile, err) } var s Status if err := json.Unmarshal([]byte(text), &s); err != nil { return nil, fmt.Errorf("the module's status file is not one: %v", err) } return &s, nil } // Roster is the mesh's machines by name, with their private addresses, as the mesh rendered them. func (m *Machine) Roster() (map[string]string, string) { text, err := m.Read(RosterFile) if err != nil { return map[string]string{}, fmt.Sprintf("the mesh's roster (%s) could not be read: %v; no node's address is known, so nothing is exported", RosterFile, err) } out := map[string]string{} for _, line := range strings.Split(text, "\n") { f := strings.Fields(line) if len(f) != 2 || strings.HasPrefix(f[0], "#") { continue } out[f[0]] = f[1] } return out, "" } // Wants is every node's wants of this machine, as the tools last took them from the bus. With none taken // yet, nothing is wanted, and nothing is exported. func (m *Machine) Wants() (map[string][]Want, string) { text, err := m.Read(WantsFile) if err != nil { return map[string][]Want{}, "no node's wants have been taken from the bus yet, so nothing is exported" } var w WantsFromBus if err := json.Unmarshal([]byte(text), &w); err != nil { return map[string][]Want{}, fmt.Sprintf("the wants taken from the bus (%s) could not be read: %v; nothing is exported", WantsFile, err) } if w.Wants == nil { w.Wants = map[string][]Want{} } return w.Wants, "" } // SharedLive says whether the kernel exports a share to every node it is meant for. func SharedLive(live []Entry, e Exported) bool { for _, g := range e.To { if !IsLive(live, e.Path, g.Address) { return false } } return true } // Stopped is the process ending. When the module was unassigned (see unassigned) the module's export file // goes and the kernel lets the shares go. A plain restart, a push or the machine shutting down leaves the // config file and the unit enabled, and the exports stay. func (m *Machine) Stopped() { ctx, cancel := context.WithTimeout(context.Background(), StopTimeout) defer cancel() if !m.unassigned(ctx) { return } // The file first: removing it needs no command, so a hanging sudo or exportfs cannot keep it. if err := m.unexport(); err != nil { loud("the module was unassigned and its export file is still there: " + err.Error() + "; remove " + ExportFile + " and run exportfs -ra as root") return } fmt.Fprintln(os.Stderr, "nfs-server: unassigned: the export file "+ExportFile+" is removed") if _, err := m.AsRootWithin(ctx, "exportfs", "-ra"); err != nil { loud("the module was unassigned and its export file is removed, but the kernel may still export its " + "shares until exportfs -ra runs as root: " + err.Error()) return } fmt.Fprintln(os.Stderr, "nfs-server: unassigned: the kernel let the shares go (exportfs -ra)") } // unassigned says whether this stop is the module being taken away: its config file is gone (the node-engine // removes resources in reverse of their record, and the config is recorded after this process), or this // process's own unit is disabled. The node-engine stops a process it removes with `systemctl disable // --now`, which disables the unit before it stops it, while a restart, a push or a shutdown leaves it // enabled — so the second sign holds even when a failed first apply recorded the process after its config // (mesh-host #62) and the config is still there. func (m *Machine) unassigned(ctx context.Context) bool { if _, err := m.Read(ConfigFile); os.IsNotExist(err) { return true } r := m.Run(ctx, "systemctl", "is-enabled", ProcessUnit) return strings.TrimSpace(r.Out) == "disabled" } // loud says, on the process's log where the service manager's journal keeps it, what an unassignment left // undone, so a person finds it rather than an export that outlived its module. func loud(what string) { fmt.Fprintln(os.Stderr, "nfs-server: UNASSIGNED, NOT CLEAN: "+what) } // Stray is what the kernel exports that is the module's and that its file no longer carries: an entry to a // client inside the private network's range — the only clients the module ever exports to — that neither // the module's file nor any other export file (/etc/exports, ZFS's own) has. What another file exports is // not the module's, whatever its client. func Stray(live, mine, foreign []Entry, rng string) []Entry { prefix, err := netip.ParsePrefix(rng) if err != nil { return nil } has := map[string]bool{} for _, e := range append(append([]Entry{}, mine...), foreign...) { has[path.Clean(e.Path)+" "+e.Client] = true } var out []Entry for _, e := range live { if has[path.Clean(e.Path)+" "+e.Client] || !insideRange(e.Client, prefix) { continue } out = append(out, e) } return out } // insideRange says whether a client — one address, or a network — lies wholly inside a range. func insideRange(client string, prefix netip.Prefix) bool { if a, err := netip.ParseAddr(client); err == nil { return prefix.Contains(a) } if p, err := netip.ParsePrefix(client); err == nil { return p.Bits() >= prefix.Bits() && prefix.Contains(p.Addr()) } return false }