Z15 zfs-tools: zfs_config, zfs_snapshots_schedule, zfs_module_events, zfs_snapshots_check and zfs_findings read what the module owns, keeps and says
zfs_config also says what the module says before it leaves a node (Z17): the next load of the kernel module uses ZFS's defaults, the three units and /etc/hostid stay. Builds: design 42 zfs-owns-zfs §28, §30, §10, A-31, A-35
This commit is contained in:
@@ -163,3 +163,45 @@ func (busLedger) Entries() ([]LedgerEntry, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// EventsStream is the module's events stream's local name in its `requires` (§25).
|
||||
const EventsStream = "events"
|
||||
|
||||
// busStream is the module's events stream through the tool runner.
|
||||
type busStream struct{}
|
||||
|
||||
// Last reads the stream from its first sequence on, page by page, and keeps the last n events of the kind.
|
||||
func (busStream) Last(n int, kind string) ([]LoggedEvent, error) {
|
||||
raw, err := stdio.BusState(buscalls.StreamFill, buscalls.StreamRequest{Stream: EventsStream})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var fill buscalls.StreamFilled
|
||||
if err := json.Unmarshal(raw, &fill); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []LoggedEvent
|
||||
from := fill.First
|
||||
for from != 0 && from <= fill.Last {
|
||||
raw, err := stdio.BusState(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: EventsStream, From: from, Limit: 500})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page buscalls.StreamPage
|
||||
if err := json.Unmarshal(raw, &page); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, msg := range page.Messages {
|
||||
k := KindOfSubject(msg.Subject)
|
||||
if kind != "" && k != kind {
|
||||
continue
|
||||
}
|
||||
out = append(out, LoggedEvent{Sequence: msg.Sequence, Kind: k, At: msg.At, Body: msg.Body})
|
||||
if len(out) > n {
|
||||
out = out[1:]
|
||||
}
|
||||
}
|
||||
from = page.Next
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -139,6 +139,10 @@ type Machine struct {
|
||||
Emit func(eventType string, body any) error
|
||||
// States is the module's states on the bus (§26); nil where this process reaches none.
|
||||
States StateStore
|
||||
// Exists says whether a path is there (a link counts); nil says no path is.
|
||||
Exists func(path string) bool
|
||||
// Log is the module's own events stream as the tools bundle reaches it; nil in a process.
|
||||
Log EventLog
|
||||
// Errors is what the health check last saw of each device's error counts; nil judges every count
|
||||
// as past.
|
||||
Errors *ErrorMemory
|
||||
@@ -146,7 +150,7 @@ type Machine struct {
|
||||
|
||||
// Here is the machine this process runs on.
|
||||
func Here() *Machine {
|
||||
return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Node: os.Getenv("MESH_NODE"), WriteFile: writeLive, Lock: FileLock, Errors: LoadErrorMemory(ErrorsFile)}
|
||||
return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Node: os.Getenv("MESH_NODE"), WriteFile: writeLive, Lock: FileLock, Exists: func(p string) bool { _, err := os.Lstat(p); return err == nil }, Errors: LoadErrorMemory(ErrorsFile)}
|
||||
}
|
||||
|
||||
// NotApplicable is the answer of a machine without ZFS: the module does not apply here.
|
||||
|
||||
@@ -67,7 +67,7 @@ func main() {
|
||||
}
|
||||
m := Here()
|
||||
m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) }
|
||||
m.States, m.Call, m.Ledger, m.Await = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await
|
||||
m.States, m.Call, m.Ledger, m.Await, m.Log = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await, busStream{}
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs.
|
||||
if err := stdio.Serve("", Tools(m)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
|
||||
@@ -37,7 +37,11 @@ func readManifest(t *testing.T) (manifest, string) {
|
||||
// which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121).
|
||||
// handTools are the tools that start or steer the scheduled run, which replace no command.
|
||||
var handTools = map[string]bool{"zfs_snapshots_run_now": true, "zfs_snapshots_prune_now": true, "zfs_config_apply": true,
|
||||
"zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true}
|
||||
"zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true,
|
||||
// The module's own read tools (§28) replace no command.
|
||||
"zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true}
|
||||
|
||||
var ownReads = map[string]bool{"zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true}
|
||||
|
||||
func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) {
|
||||
m, raw := readManifest(t)
|
||||
@@ -45,7 +49,7 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T
|
||||
for _, tl := range Tools(newFake(t).machine()) {
|
||||
served = append(served, tl.Name)
|
||||
want := "(r)"
|
||||
if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || handTools[tl.Name] {
|
||||
if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || (handTools[tl.Name] && !ownReads[tl.Name]) {
|
||||
want = "(a)"
|
||||
}
|
||||
if !strings.HasSuffix(tl.Description, want) {
|
||||
|
||||
@@ -106,6 +106,33 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
return map[string]any{"recorded": true}, nil
|
||||
},
|
||||
},
|
||||
readTool("zfs_config", "Every value the module owns with its config value, what runs on the node, and how it stands: not managed, as set, "+
|
||||
"not applied yet, waiting (on what, since when), refused (why), failed, or taking effect at the next load of the kernel module; the "+
|
||||
"package's files the module never writes (with a .pacnew beside them) and what the module says before it leaves a node: the next load of "+
|
||||
"the kernel module uses ZFS's defaults, the three units stay, /etc/hostid stays. Changes nothing. (r)", nil,
|
||||
func(map[string]any) (any, error) { return m.ConfigNow(ctx) }),
|
||||
readTool("zfs_snapshots_schedule", "The schedule of the scheduled ZFS snapshots: the keep counts, when each kind next falls due, whether it is paused, "+
|
||||
"the entries past their keep count the next run would remove, and the ledger `{\"ledger\": [...]}` of every scheduled ZFS snapshot the run took "+
|
||||
"and did not remove. The scheduled run reads the ledger from here. Changes nothing. (r)", nil,
|
||||
func(map[string]any) (any, error) { return m.SnapshotSchedule() }),
|
||||
readTool("zfs_module_events", "The module's own events, newest first, from its events stream (config.applied, config.refused, config.waiting, "+
|
||||
"snapshot.taken.person and .schedule, snapshot.removed.person and .schedule, snapshot.skipped.person and .schedule, run.skipped, scrub.requested), "+
|
||||
"the last N of one kind or of all, with how many of each kind. Not ZFS's own events: zfs_events reads those. Changes nothing. (r)",
|
||||
map[string]any{"last": map[string]any{"type": "integer", "description": "how many to show (default 20, 1 to 500)"},
|
||||
"kind": map[string]any{"type": "string", "description": "one kind of event (optional: every kind when left out)"}},
|
||||
func(a map[string]any) (any, error) {
|
||||
n, err := whole(a, "last", 20, 1, 500)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.ModuleEvents(n, text(a, "kind"))
|
||||
}),
|
||||
readTool("zfs_snapshots_check", "Compares the ledger with what ZFS holds: names each ledger entry whose ZFS snapshot is gone and each ZFS snapshot "+
|
||||
"named mesh- that the ledger does not hold. Changes nothing. (r)", nil,
|
||||
func(map[string]any) (any, error) { return m.CheckSnapshots(ctx) }),
|
||||
readTool("zfs_findings", "The event follower's findings: one key for each pool or device that is not ONLINE, deleted when it is ONLINE again or "+
|
||||
"leaves the pool. Changes nothing. (r)", nil,
|
||||
func(map[string]any) (any, error) { return m.FindingsNow() }),
|
||||
{
|
||||
Name: "zfs_pools",
|
||||
Description: "Every imported pool's health in a line: its state (ONLINE, DEGRADED, FAULTED, …) with status's own " +
|
||||
@@ -311,3 +338,8 @@ func handTool(m *Machine, name, act, description string) stdio.Tool {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// readTool is one of the module's own read tools.
|
||||
func readTool(name, description string, input map[string]any, run func(map[string]any) (any, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: name, Description: description, Input: input, Run: run}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The module's own read tools (design 42 zfs-owns-zfs §28, §30): what is owned and how it stands, the
|
||||
// schedule of the scheduled ZFS snapshots with the ledger, the module's events, the ledger against ZFS, and the
|
||||
// follower's findings. None of them changes anything.
|
||||
|
||||
// configKeys are the config values the module owns, in the order they are said.
|
||||
var configKeys = []string{"pool", "scrub-cadence", "scrub-day", "scrub-time", "arc-max", "arc-min", "module-options", "hostid",
|
||||
"zed", "zed-handlers", "trim-schedule", "trim-day", "trim-time", "cache-devices", "log-devices", "log-mirror",
|
||||
"pool-properties", "dataset-properties", "snapshots"}
|
||||
|
||||
// appliedKeys are the values the apply converges (the others are the node-engine's files or the run's schedule):
|
||||
// one of them set and with no key in the state `applied` has not been applied yet.
|
||||
var appliedKeys = map[string]bool{"arc-max": true, "arc-min": true, "module-options": true, "hostid": true, "cache-devices": true,
|
||||
"log-devices": true, "log-mirror": true, "pool-properties": true, "dataset-properties": true}
|
||||
|
||||
// The states a value is shown in (§28).
|
||||
const (
|
||||
StateNotManaged = "not managed"
|
||||
StateAsSet = "as set"
|
||||
StateNotYet = "not applied yet"
|
||||
StateWaiting = "waiting"
|
||||
StateRefused = "refused"
|
||||
StateFailed = "failed"
|
||||
StateAwaitingLoad = "takes effect at the next load of the kernel module"
|
||||
)
|
||||
|
||||
// ConfigValue is one owned config value as zfs_config shows it.
|
||||
type ConfigValue struct {
|
||||
Key string `json:"key"`
|
||||
// Value is what the config file holds, absent when the value is not managed.
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
// Running is what runs on the node, where it is read live; else what the last apply read back.
|
||||
Running string `json:"running,omitempty"`
|
||||
State string `json:"state"`
|
||||
Why string `json:"why,omitempty"`
|
||||
WaitsOn string `json:"waits-on,omitempty"`
|
||||
Since string `json:"since,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// ConfigFileState is one of the package's files the module never writes (§10).
|
||||
type ConfigFileState struct {
|
||||
Path string `json:"path"`
|
||||
Present bool `json:"present"`
|
||||
// Pacnew says a `.pacnew` lies beside it: the package shipped a newer file the module did not take.
|
||||
Pacnew bool `json:"pacnew,omitempty"`
|
||||
}
|
||||
|
||||
// ConfigAnswer is what zfs_config answers.
|
||||
type ConfigAnswer struct {
|
||||
Values []ConfigValue `json:"values"`
|
||||
Files []ConfigFileState `json:"files"`
|
||||
Leaving []string `json:"leaving"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
// packageFiles are the files of §10: the package's, which the module never writes, and ZFS's own caches.
|
||||
var packageFiles = []string{"/etc/default/zfs", "/etc/zfs/zed.d", "/etc/zfs/zed.d/zed.rc", "/etc/zfs/zpool.cache", "/etc/zfs/zfs-list.cache"}
|
||||
|
||||
// leavingSaid is what the module says before it leaves a node (§30).
|
||||
var leavingSaid = []string{
|
||||
"When the module leaves, the next load of the kernel module uses ZFS's own defaults: zfs.conf is removed.",
|
||||
"The units zfs-import-cache.service, zfs-mount.service and zfs.target stay running and enabled; they are never stopped or disabled.",
|
||||
"/etc/hostid stays: it is no resource the node-engine removes, and removing it would change the node's hostid, so that the pool's next import could need zpool import -f.",
|
||||
"Pool and dataset properties and devices stay as the last apply set them.",
|
||||
"The distribution's scrub and trim timers are put back as the module found them.",
|
||||
}
|
||||
|
||||
func (m *Machine) exists(path string) bool { return m.Exists != nil && m.Exists(path) }
|
||||
|
||||
func rawManaged(raw json.RawMessage) bool {
|
||||
t := strings.TrimSpace(string(raw))
|
||||
switch t {
|
||||
case "", "null", `""`, "[]", "{}":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// live is a kernel parameter read as it runs.
|
||||
func (m *Machine) live(name string) (string, bool) {
|
||||
b, err := m.ReadFile(ParametersDir + name)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(string(b)), true
|
||||
}
|
||||
|
||||
// worse orders the outcomes of a value's keys: the one a person must see first.
|
||||
var worse = map[string]int{OutcomeApplied: 1, OutcomeAwaitingLoad: 2, OutcomeWaiting: 3, OutcomeFailed: 4, OutcomeRefused: 5}
|
||||
|
||||
// ConfigNow shows every value the module owns: what the config file holds, what runs, how it stands, and the
|
||||
// files it never writes.
|
||||
func (m *Machine) ConfigNow(ctx context.Context) (ConfigAnswer, error) {
|
||||
a := ConfigAnswer{Values: []ConfigValue{}, Files: []ConfigFileState{}, Leaving: leavingSaid, Said: []string{}}
|
||||
text, err := m.ReadFile(ConfigFile)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("%s could not be read: %v", ConfigFile, err)
|
||||
}
|
||||
raw := map[string]json.RawMessage{}
|
||||
if err := json.Unmarshal(text, &raw); err != nil {
|
||||
return a, fmt.Errorf("%s is not a JSON object: %v", ConfigFile, err)
|
||||
}
|
||||
applied := map[string]AppliedEntry{}
|
||||
if m.States != nil {
|
||||
keys, err := m.States.Keys(StateApplied)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the state %s could not be read: %v", StateApplied, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if v, ok, _ := m.States.Get(StateApplied, k); ok {
|
||||
var e AppliedEntry
|
||||
if json.Unmarshal(v, &e) == nil {
|
||||
applied[k] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range configKeys {
|
||||
v := ConfigValue{Key: key, State: StateNotManaged}
|
||||
if !rawManaged(raw[key]) {
|
||||
a.Values = append(a.Values, v)
|
||||
continue
|
||||
}
|
||||
v.Value = raw[key]
|
||||
v.State = StateAsSet
|
||||
if appliedKeys[key] {
|
||||
v.State = StateNotYet
|
||||
}
|
||||
best := 0
|
||||
var afters []string
|
||||
for _, k := range sortedKeys(applied) {
|
||||
if first, _, _ := strings.Cut(k, "."); first != key {
|
||||
continue
|
||||
}
|
||||
e := applied[k]
|
||||
if e.After != "" {
|
||||
afters = append(afters, k+"="+e.After)
|
||||
}
|
||||
if worse[e.Outcome] > best {
|
||||
best = worse[e.Outcome]
|
||||
v.Why, v.WaitsOn = e.Why, e.WaitsOn
|
||||
v.Since = ""
|
||||
if !e.Since.IsZero() {
|
||||
v.Since = e.Since.Format(time.RFC3339)
|
||||
}
|
||||
switch e.Outcome {
|
||||
case OutcomeApplied:
|
||||
v.State = StateAsSet
|
||||
case OutcomeAwaitingLoad:
|
||||
v.State = StateAwaitingLoad
|
||||
case OutcomeWaiting:
|
||||
v.State = StateWaiting
|
||||
case OutcomeFailed:
|
||||
v.State = StateFailed
|
||||
case OutcomeRefused:
|
||||
v.State = StateRefused
|
||||
}
|
||||
}
|
||||
}
|
||||
v.Running = strings.Join(afters, ", ")
|
||||
switch key {
|
||||
case "arc-max", "arc-min":
|
||||
if n, ok := m.live("zfs_" + strings.ReplaceAll(key, "-", "_")); ok {
|
||||
v.Running = n + " bytes"
|
||||
}
|
||||
case "hostid":
|
||||
if b, err := m.ReadFile(HostIDFile); err == nil {
|
||||
if id, ok := nativeHostID(b); ok {
|
||||
v.Running = fmt.Sprintf("%08x", id)
|
||||
}
|
||||
}
|
||||
v.Note = "a change of the node's ZFS id needs a check of the pool's import at the next boot; the module never reboots"
|
||||
}
|
||||
a.Values = append(a.Values, v)
|
||||
}
|
||||
for _, p := range packageFiles {
|
||||
f := ConfigFileState{Path: p, Present: m.exists(p), Pacnew: m.exists(p + ".pacnew")}
|
||||
a.Files = append(a.Files, f)
|
||||
if f.Pacnew {
|
||||
a.Said = append(a.Said, p+".pacnew lies beside it: the package shipped a newer file the module does not take.")
|
||||
}
|
||||
}
|
||||
a.Said = append([]string{fmt.Sprintf("%d of %d values are set.", countSet(a.Values), len(a.Values))}, a.Said...)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func countSet(vs []ConfigValue) int {
|
||||
n := 0
|
||||
for _, v := range vs {
|
||||
if v.State != StateNotManaged {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ScheduleAnswer is what zfs_snapshots_schedule answers.
|
||||
type ScheduleAnswer struct {
|
||||
Pool string `json:"pool,omitempty"`
|
||||
Keeps map[string]int `json:"keeps"`
|
||||
Paused map[string]any `json:"paused,omitempty"`
|
||||
// Next is the moment each kind with a keep count next falls due; the run fires every quarter of an hour, so the
|
||||
// ZFS snapshot is taken at the first run at or after it.
|
||||
Next map[string]string `json:"next"`
|
||||
// WouldRemove are the ledger's entries past their keep count: what the next run removes, if it passes its checks.
|
||||
WouldRemove []LedgerEntry `json:"would-remove"`
|
||||
Ledger []LedgerEntry `json:"ledger"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
// nextDue is the first moment after now at which a kind falls due, in now's time zone.
|
||||
func nextDue(kind string, now time.Time) time.Time {
|
||||
day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
||||
switch kind {
|
||||
case "hourly":
|
||||
return now.Truncate(time.Hour).Add(time.Hour)
|
||||
case "weekly":
|
||||
d := day.AddDate(0, 0, 1)
|
||||
for d.Weekday() != time.Monday {
|
||||
d = d.AddDate(0, 0, 1)
|
||||
}
|
||||
return d
|
||||
case "monthly":
|
||||
return time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, now.Location())
|
||||
}
|
||||
return day.AddDate(0, 0, 1)
|
||||
}
|
||||
|
||||
// wouldRemove are the entries past their dataset's and kind's keep count; a kind with 0 removes nothing (A-35).
|
||||
func wouldRemove(entries []LedgerEntry, k *Keeps) []LedgerEntry {
|
||||
groups := map[string][]LedgerEntry{}
|
||||
for _, e := range entries {
|
||||
g := e.Dataset + " " + e.Kind
|
||||
groups[g] = append(groups[g], e)
|
||||
}
|
||||
out := []LedgerEntry{}
|
||||
for _, g := range sortedKeys(groups) {
|
||||
list := groups[g]
|
||||
if n := k.Of(list[0].Kind); n > 0 {
|
||||
out = append(out, surplus(list, n)...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SnapshotSchedule shows the schedule of the scheduled ZFS snapshots, its next run, what the next run would remove and
|
||||
// the ledger. The ledger is shown even when the config says nothing of it: the scheduled run reads it from here.
|
||||
func (m *Machine) SnapshotSchedule() (ScheduleAnswer, error) {
|
||||
a := ScheduleAnswer{Keeps: map[string]int{}, Next: map[string]string{}, WouldRemove: []LedgerEntry{}, Ledger: []LedgerEntry{}, Said: []string{}}
|
||||
if m.Ledger == nil {
|
||||
return a, fmt.Errorf("this process reaches no ledger")
|
||||
}
|
||||
entries, err := m.Ledger.Entries()
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err)
|
||||
}
|
||||
if entries != nil {
|
||||
a.Ledger = entries
|
||||
}
|
||||
if m.States != nil {
|
||||
if raw, ok, err := m.States.Get(StateSchedule, KeyPaused); err == nil && ok {
|
||||
_ = json.Unmarshal(raw, &a.Paused)
|
||||
a.Said = append(a.Said, "The schedule is paused: nothing is taken or removed until zfs_snapshots_resume.")
|
||||
}
|
||||
}
|
||||
text, err := m.ReadFile(ConfigFile)
|
||||
if err != nil {
|
||||
a.Said = append(a.Said, ConfigFile+" could not be read: "+err.Error())
|
||||
return a, nil
|
||||
}
|
||||
c, refused := ParseConfig(text, m.memMiB())
|
||||
switch {
|
||||
case len(refused) > 0:
|
||||
a.Said = append(a.Said, "The config is refused ("+refused[0].Key+" "+refused[0].Reason+"): the schedule is not known.")
|
||||
case c.Snapshots == nil:
|
||||
a.Said = append(a.Said, "snapshots is not managed: nothing is taken or removed.")
|
||||
case c.Snapshots.None:
|
||||
a.Pool = c.Pool
|
||||
a.Said = append(a.Said, "snapshots is none: nothing is taken, and the scheduled ZFS snapshots that exist are kept.")
|
||||
default:
|
||||
a.Pool = c.Pool
|
||||
for _, kind := range keepKinds {
|
||||
a.Keeps[kind] = c.Snapshots.Of(kind)
|
||||
if a.Keeps[kind] > 0 {
|
||||
a.Next[kind] = nextDue(kind, m.Now()).Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
a.WouldRemove = wouldRemove(a.Ledger, c.Snapshots)
|
||||
a.Said = append(a.Said, fmt.Sprintf("%d scheduled ZFS snapshots are in the ledger; the next run would remove %d.", len(a.Ledger), len(a.WouldRemove)))
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// LoggedEvent is one event of the module's stream.
|
||||
type LoggedEvent struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"at"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
}
|
||||
|
||||
// EventLog is the module's own events stream as the tools bundle reaches it (§25).
|
||||
type EventLog interface {
|
||||
// Last are the last n events, oldest first, only those of the kind when one is named.
|
||||
Last(n int, kind string) ([]LoggedEvent, error)
|
||||
}
|
||||
|
||||
// EventKinds are the module's kinds of event (§25).
|
||||
var EventKinds = []string{EventApplied, EventRefused, EventWaiting, EventTakenPerson, EventTakenSchedule, EventRemovedPerson,
|
||||
EventRemovedSched, EventSkippedPerson, EventSkippedSched, EventRunSkipped, EventScrubRequested}
|
||||
|
||||
// KindOfSubject is the kind an event's subject ends in.
|
||||
func KindOfSubject(subject string) string {
|
||||
for _, k := range EventKinds {
|
||||
if subject == k || strings.HasSuffix(subject, "."+k) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
return subject
|
||||
}
|
||||
|
||||
// ModuleEventsAnswer is what zfs_module_events answers.
|
||||
type ModuleEventsAnswer struct {
|
||||
Events []LoggedEvent `json:"events"`
|
||||
ByKind map[string]int `json:"by-kind"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
// ModuleEvents are the last n events of the module's stream, newest first, of one kind when it is named.
|
||||
func (m *Machine) ModuleEvents(n int, kind string) (ModuleEventsAnswer, error) {
|
||||
a := ModuleEventsAnswer{Events: []LoggedEvent{}, ByKind: map[string]int{}, Said: []string{}}
|
||||
if kind != "" {
|
||||
known := false
|
||||
for _, k := range EventKinds {
|
||||
known = known || k == kind
|
||||
}
|
||||
if !known {
|
||||
return a, fmt.Errorf("%q is not a kind of this module's events: %s", kind, strings.Join(EventKinds, ", "))
|
||||
}
|
||||
}
|
||||
if m.Log == nil {
|
||||
return a, fmt.Errorf("this process reaches no events stream")
|
||||
}
|
||||
evs, err := m.Log.Last(n, kind)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the events stream could not be read: %v", err)
|
||||
}
|
||||
for i := len(evs) - 1; i >= 0; i-- {
|
||||
a.Events = append(a.Events, evs[i])
|
||||
a.ByKind[evs[i].Kind]++
|
||||
}
|
||||
a.Said = append(a.Said, fmt.Sprintf("%d events shown, newest first; the stream keeps 30 days.", len(a.Events)))
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// CheckAnswer is what zfs_snapshots_check answers.
|
||||
type CheckAnswer struct {
|
||||
// Gone are the ledger's entries whose ZFS snapshot ZFS no longer has.
|
||||
Gone []LedgerEntry `json:"gone"`
|
||||
// Unrecorded are the snapshots named mesh- that ZFS has and the ledger does not.
|
||||
Unrecorded []string `json:"unrecorded"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
// CheckSnapshots compares the ledger with what ZFS holds; it changes nothing.
|
||||
func (m *Machine) CheckSnapshots(ctx context.Context) (CheckAnswer, error) {
|
||||
a := CheckAnswer{Gone: []LedgerEntry{}, Unrecorded: []string{}, Said: []string{}}
|
||||
if m.Ledger == nil {
|
||||
return a, fmt.Errorf("this process reaches no ledger")
|
||||
}
|
||||
entries, err := m.Ledger.Entries()
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err)
|
||||
}
|
||||
snaps, err := m.Snapshots(ctx)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, s := range snaps {
|
||||
have[s.Name] = true
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
full := e.Dataset + "@" + e.Name
|
||||
recorded[full] = true
|
||||
if !have[full] {
|
||||
a.Gone = append(a.Gone, e)
|
||||
}
|
||||
}
|
||||
for _, s := range snaps {
|
||||
if strings.HasPrefix(s.Snap, MeshPrefix) && !recorded[s.Name] {
|
||||
a.Unrecorded = append(a.Unrecorded, s.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(a.Unrecorded)
|
||||
a.Said = append(a.Said, fmt.Sprintf("%d ledger entries, %d with no ZFS snapshot, %d mesh- snapshots not in the ledger. Nothing was changed.",
|
||||
len(entries), len(a.Gone), len(a.Unrecorded)))
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// FindingsAnswer is what zfs_findings answers.
|
||||
type FindingsAnswer struct {
|
||||
Findings []FindingKey `json:"findings"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
// FindingKey is one key of the follower's state `findings`.
|
||||
type FindingKey struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
// FindingsNow are the follower's findings: one key per pool or device not ONLINE.
|
||||
func (m *Machine) FindingsNow() (FindingsAnswer, error) {
|
||||
a := FindingsAnswer{Findings: []FindingKey{}, Said: []string{}}
|
||||
if m.States == nil {
|
||||
return a, fmt.Errorf("this process reaches no state")
|
||||
}
|
||||
keys, err := m.States.Keys(StateFindings)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the state %s could not be read: %v", StateFindings, err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if v, ok, err := m.States.Get(StateFindings, k); err == nil && ok {
|
||||
a.Findings = append(a.Findings, FindingKey{Key: k, Value: v})
|
||||
}
|
||||
}
|
||||
if len(a.Findings) == 0 {
|
||||
a.Said = append(a.Said, "No pool or device is not ONLINE: the follower holds no finding.")
|
||||
} else {
|
||||
a.Said = append(a.Said, fmt.Sprintf("%d findings: a pool or device not ONLINE.", len(a.Findings)))
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type staticLedger struct {
|
||||
entries []LedgerEntry
|
||||
err error
|
||||
}
|
||||
|
||||
func (l staticLedger) Apply(string, string, any, string) error { return nil }
|
||||
func (l staticLedger) Entries() ([]LedgerEntry, error) { return l.entries, l.err }
|
||||
|
||||
type staticLog struct {
|
||||
evs []LoggedEvent
|
||||
err error
|
||||
}
|
||||
|
||||
func (l staticLog) Last(n int, kind string) ([]LoggedEvent, error) {
|
||||
var out []LoggedEvent
|
||||
for _, e := range l.evs {
|
||||
if kind == "" || e.Kind == kind {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if len(out) > n {
|
||||
out = out[len(out)-n:]
|
||||
}
|
||||
return out, l.err
|
||||
}
|
||||
|
||||
// §28, §30, §10, §29: zfs_config shows each owned value with its config value, what runs, and how it stands
|
||||
// (not managed, as set, not applied yet, waiting, refused, awaiting a load), the package's files with a .pacnew
|
||||
// beside them, and what the module says before it leaves a node.
|
||||
func TestZfsConfigShowsEveryValueHowItStandsAndWhatTheModuleSaysBeforeLeaving(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := f.machine()
|
||||
st := memStates{}
|
||||
m.States = st
|
||||
f.files[ConfigFile] = `{"pool":"tank","arc-max":4096,"module-options":["zfs_txg_timeout=5"],"hostid":"0a0b0c0d","cache-devices":["x"],
|
||||
"log-devices":[],"zed":"on","trim-schedule":"","snapshots":{"daily":3}}`
|
||||
f.files[ParametersDir+"zfs_arc_max"] = "4294967296\n"
|
||||
hb := make([]byte, 4)
|
||||
binary.NativeEndian.PutUint32(hb, 0x0a0b0c0d)
|
||||
f.files[HostIDFile] = string(hb)
|
||||
m.Exists = func(p string) bool { return p == "/etc/default/zfs" || p == "/etc/default/zfs.pacnew" }
|
||||
st.Put(StateApplied, "arc-max", AppliedEntry{Before: "1", After: "4294967296", Outcome: OutcomeApplied})
|
||||
st.Put(StateApplied, "module-options.zfs_txg_timeout", AppliedEntry{After: "5", Outcome: OutcomeAwaitingLoad})
|
||||
st.Put(StateApplied, "hostid", AppliedEntry{After: "0a0b0c0d", Outcome: OutcomeRefused, Why: "the pool's import would need -f"})
|
||||
st.Put(StateApplied, "cache-devices", AppliedEntry{Outcome: OutcomeWaiting, WaitsOn: "a scrub of tank", Since: f.now})
|
||||
st.Put(StateApplied, "trim-day", AppliedEntry{Outcome: OutcomeApplied}) // a value gone back to not managed is not shown
|
||||
a, err := m.ConfigNow(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]ConfigValue{}
|
||||
for _, v := range a.Values {
|
||||
by[v.Key] = v
|
||||
}
|
||||
want := map[string]string{"pool": StateAsSet, "arc-max": StateAsSet, "module-options": StateAwaitingLoad, "hostid": StateRefused,
|
||||
"cache-devices": StateWaiting, "log-devices": StateNotManaged, "zed": StateAsSet, "trim-schedule": StateNotManaged,
|
||||
"trim-day": StateNotManaged, "snapshots": StateAsSet, "arc-min": StateNotManaged}
|
||||
for k, w := range want {
|
||||
if by[k].State != w {
|
||||
t.Errorf("%s: %q, want %q", k, by[k].State, w)
|
||||
}
|
||||
}
|
||||
if len(a.Values) != len(configKeys) {
|
||||
t.Errorf("%d values for %d keys", len(a.Values), len(configKeys))
|
||||
}
|
||||
if by["arc-max"].Running != "4294967296 bytes" || by["hostid"].Running != "0a0b0c0d" || by["hostid"].Why == "" ||
|
||||
!strings.Contains(by["hostid"].Note, "check of the pool's import") || by["cache-devices"].WaitsOn != "a scrub of tank" || by["cache-devices"].Since == "" {
|
||||
t.Errorf("values: %+v", by)
|
||||
}
|
||||
if by["arc-min"].Value != nil {
|
||||
t.Errorf("an unmanaged value shows %s", by["arc-min"].Value)
|
||||
}
|
||||
pacnew := false
|
||||
for _, fl := range a.Files {
|
||||
pacnew = pacnew || (fl.Path == "/etc/default/zfs" && fl.Present && fl.Pacnew)
|
||||
}
|
||||
all := strings.Join(a.Leaving, " ")
|
||||
if !pacnew || !strings.Contains(strings.Join(a.Said, " "), "/etc/default/zfs.pacnew") || len(a.Files) != len(packageFiles) {
|
||||
t.Errorf("files %+v said %v", a.Files, a.Said)
|
||||
}
|
||||
for _, s := range []string{"ZFS's own defaults", "zfs-import-cache.service, zfs-mount.service and zfs.target stay", "/etc/hostid stays"} {
|
||||
if !strings.Contains(all, s) {
|
||||
t.Errorf("leaving does not say %q: %v", s, a.Leaving)
|
||||
}
|
||||
}
|
||||
delete(f.files, ConfigFile)
|
||||
if _, err := m.ConfigNow(context.Background()); err == nil {
|
||||
t.Error("an unreadable config file was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// §28, A-31: zfs_snapshots_schedule answers the keep counts, each kind's next moment, the pause, what the next
|
||||
// run would remove (a keep count of 0 removes nothing, A-35) and the ledger as `{"ledger": [...]}`, which the run reads.
|
||||
func TestSnapshotsScheduleAnswersTheLedgerTheRunReads(t *testing.T) {
|
||||
e := func(h int) LedgerEntry {
|
||||
return LedgerEntry{"tank", "mesh-hourly-x" + string(rune('0'+h)), "hourly", time.Date(2026, 10, 9, h, 0, 0, 0, time.UTC).Format(time.RFC3339)}
|
||||
}
|
||||
old := LedgerEntry{"tank", "mesh-daily-old", "daily", "2026-09-01T00:00:00Z"}
|
||||
cases := []struct {
|
||||
name, conf string
|
||||
l staticLedger
|
||||
paused bool
|
||||
remove int
|
||||
next map[string]string
|
||||
said string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "counts and surplus", conf: `{"pool":"tank","snapshots":{"hourly":2,"daily":0}}`, l: staticLedger{entries: []LedgerEntry{e(1), e(2), e(3), old}},
|
||||
remove: 1, next: map[string]string{"hourly": "2026-10-09T13:00:00"}, said: "would remove 1"},
|
||||
{name: "weekly and monthly", conf: `{"pool":"tank","snapshots":{"weekly":1,"monthly":1,"daily":1}}`, next: map[string]string{
|
||||
"daily": "2026-10-10T00:00:00", "weekly": "2026-10-12T00:00:00", "monthly": "2026-11-01T00:00:00"}},
|
||||
{name: "paused", conf: `{"pool":"tank","snapshots":{"hourly":1}}`, paused: true, said: "paused"},
|
||||
{name: "none keeps what exists", conf: `{"pool":"tank","snapshots":{"none":true}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "are kept"},
|
||||
{name: "not managed", conf: `{"pool":"tank"}`, said: "not managed"},
|
||||
{name: "refused config still answers the ledger", conf: `{"snapshots":{"hourly":999}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "refused"},
|
||||
{name: "ledger unreadable", conf: `{}`, l: staticLedger{err: errors.New("no bus")}, wantErr: true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
f := newFake(t)
|
||||
f.now = time.Date(2026, 10, 9, 12, 30, 0, 0, time.UTC) // a Friday
|
||||
m := f.machine()
|
||||
st := memStates{}
|
||||
m.States, m.Ledger = st, c.l
|
||||
f.files[ConfigFile] = c.conf
|
||||
if c.paused {
|
||||
st.Put(StateSchedule, KeyPaused, map[string]string{"by": "x", "why": "y"})
|
||||
}
|
||||
a, err := m.SnapshotSchedule()
|
||||
if (err != nil) != c.wantErr {
|
||||
t.Errorf("%s: %v", c.name, err)
|
||||
continue
|
||||
}
|
||||
if c.wantErr {
|
||||
continue
|
||||
}
|
||||
if len(a.WouldRemove) != c.remove || (c.paused && a.Paused == nil) || !strings.Contains(strings.Join(a.Said, " "), c.said) {
|
||||
t.Errorf("%s: %+v", c.name, a)
|
||||
}
|
||||
for k, v := range c.next {
|
||||
if !strings.HasPrefix(a.Next[k], v) {
|
||||
t.Errorf("%s: next %s %q, want %q", c.name, k, a.Next[k], v)
|
||||
}
|
||||
}
|
||||
// The run reads the answer as it reads the ledger.
|
||||
raw, _ := json.Marshal(a)
|
||||
r := f.machine()
|
||||
r.Call = func(key string, _ any) (json.RawMessage, error) { return raw, nil }
|
||||
got, err := r.ledgerEntries()
|
||||
if err != nil || len(got) != len(c.l.entries) {
|
||||
t.Errorf("%s: the run reads %v, %v", c.name, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// §28: zfs_module_events answers the last N events of the stream, newest first, of one kind when named, with how
|
||||
// many of each; a kind the module does not say is refused.
|
||||
func TestModuleEventsAreTheStreamsLastNNewestFirstByKind(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := f.machine()
|
||||
at := func(i int) time.Time { return time.Date(2026, 10, 9, 10, i, 0, 0, time.UTC) }
|
||||
m.Log = staticLog{evs: []LoggedEvent{{1, EventApplied, at(1), nil}, {2, EventRefused, at(2), nil}, {3, EventApplied, at(3), nil}, {4, EventRunSkipped, at(4), nil}}}
|
||||
a, err := m.ModuleEvents(3, "")
|
||||
if err != nil || len(a.Events) != 3 || a.Events[0].Sequence != 4 || a.ByKind[EventApplied] != 1 || a.ByKind[EventRunSkipped] != 1 {
|
||||
t.Errorf("last 3: %+v %v", a, err)
|
||||
}
|
||||
if a, err := m.ModuleEvents(5, EventApplied); err != nil || len(a.Events) != 2 || a.Events[0].Sequence != 3 {
|
||||
t.Errorf("one kind: %+v %v", a, err)
|
||||
}
|
||||
if _, err := m.ModuleEvents(5, "config.nothing"); err == nil {
|
||||
t.Error("an unknown kind was answered")
|
||||
}
|
||||
m.Log = staticLog{err: errors.New("no bus")}
|
||||
if _, err := m.ModuleEvents(5, ""); err == nil {
|
||||
t.Error("a stream that cannot be read was answered")
|
||||
}
|
||||
for subject, kind := range map[string]string{"zfs.snapshot.taken.person": EventTakenPerson, "snapshot.taken.schedule": EventTakenSchedule, "other": "other"} {
|
||||
if KindOfSubject(subject) != kind {
|
||||
t.Errorf("%s is %s", subject, KindOfSubject(subject))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// §28: zfs_snapshots_check names each ledger entry whose ZFS snapshot is gone and each mesh- snapshot the ledger
|
||||
// does not hold, and changes nothing.
|
||||
func TestSnapshotsCheckNamesWhatTheLedgerAndZFSDoNotShare(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.on(line("zfs", SnapshotsArgv()), "tank@mesh-hourly-20261009T1000Z\t1791400000\t1\t2\ntank@mesh-hourly-20261009T1100Z\t1791400000\t1\t2\ntank@by-hand\t1791400000\t1\t2\n")
|
||||
f.uid = 0
|
||||
m := f.machine()
|
||||
m.Ledger = staticLedger{entries: []LedgerEntry{{"tank", "mesh-hourly-20261009T1000Z", "hourly", "2026-10-09T10:00:00Z"},
|
||||
{"tank", "mesh-hourly-20261009T0900Z", "hourly", "2026-10-09T09:00:00Z"}}}
|
||||
a, err := m.CheckSnapshots(context.Background())
|
||||
if err != nil || len(a.Gone) != 1 || a.Gone[0].Name != "mesh-hourly-20261009T0900Z" ||
|
||||
len(a.Unrecorded) != 1 || a.Unrecorded[0] != "tank@mesh-hourly-20261009T1100Z" {
|
||||
t.Errorf("%+v %v", a, err)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if strings.Contains(c, "destroy") || strings.Contains(c, " set ") || strings.Contains(c, "snapshot --") {
|
||||
t.Errorf("a check changed something: %s", c)
|
||||
}
|
||||
}
|
||||
m.Ledger = staticLedger{err: errors.New("no bus")}
|
||||
if _, err := m.CheckSnapshots(context.Background()); err == nil {
|
||||
t.Error("a ledger that cannot be read was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// §27, §28: zfs_findings shows the follower's keys, none when every device is ONLINE.
|
||||
func TestFindingsAreTheFollowersKeys(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := f.machine()
|
||||
st := memStates{}
|
||||
m.States = st
|
||||
if a, err := m.FindingsNow(); err != nil || len(a.Findings) != 0 || !strings.Contains(a.Said[0], "No pool or device") {
|
||||
t.Errorf("%+v %v", a, err)
|
||||
}
|
||||
st.Put(StateFindings, "pool.tank.sda", map[string]string{"summary": "sda is FAULTED"})
|
||||
if a, err := m.FindingsNow(); err != nil || len(a.Findings) != 1 || a.Findings[0].Key != "pool.tank.sda" {
|
||||
t.Errorf("%+v %v", a, err)
|
||||
}
|
||||
m.States = nil
|
||||
if _, err := m.FindingsNow(); err == nil {
|
||||
t.Error("no state, no answer")
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,11 @@
|
||||
"zfs_snapshots_pause",
|
||||
"zfs_snapshots_resume",
|
||||
"zfs_snapshots_record",
|
||||
"zfs_config",
|
||||
"zfs_snapshots_schedule",
|
||||
"zfs_module_events",
|
||||
"zfs_snapshots_check",
|
||||
"zfs_findings",
|
||||
"zfs_io",
|
||||
"zfs_arc",
|
||||
"zfs_events",
|
||||
|
||||
Reference in New Issue
Block a user