diff --git a/modules/zfs/README.md b/modules/zfs/README.md new file mode 100644 index 00000000..84589517 --- /dev/null +++ b/modules/zfs/README.md @@ -0,0 +1,107 @@ +# zfs + +The mesh's view of ZFS on a machine: every pool's health, layout, space, scrubs, I/O, cache and events, +every dataset with its space and snapshots, and one act, a named snapshot of one dataset. It is for the +home server, whose pool holds the operator's media library, which the mesh does not own (novox/hq ADR +0051). So the module reads, and the one thing it does only adds. + +## What it does not do + +There is no destroy, rollback, set, send, receive, rename, clone or any other change. Each of those changes +or removes the operator's data and needs a decision of its own first. `zfs_snapshot` is the only tool that +changes anything, and it only adds a snapshot. + +This is checked by `TestTheBundleRunsOnlyReadingVerbsAndSnapshot`. Every command the bundle runs is built +by one of its argument functions, and each function is held to its verb: `list`, `get`, `status`, +`iostat`, `events`, `history` or `snapshot`. A call site that builds its own argument list fails the test. + +It installs nothing. ZFS belongs to the machine, and its kernel module is built against that machine's +kernel. On a machine without ZFS, every tool answers `applicable: false` with the reason, and the health +check is unhealthy and says to unassign the module, because a module put where it does nothing must not +look healthy. + +It holds no seat. A new `node-*` seat is the controller's to define (novox/hq ADR 0121, ADR 0246). This +is a module with tools, like `disk-load` and `sensors`. If the mesh later gives a machine's storage a +seat, these tools become its verbs. + +## Tools + +The tools run as the operator account. What ZFS lets any account read runs as that account. What ZFS +keeps for root (events, history and the snapshot itself) goes through `sudo -n` with a fixed argument +list, never a shell. Sizes are whole bytes, read under `-p`, and each answer also says its findings in +lines under `said`. + +| tool | what | replaces | +|---|---|---| +| `zfs_datasets` | Every filesystem and volume: mountpoint, used, available, referenced, compression ratio, sharenfs, and the number of snapshots with the space they hold together (`usedbysnapshots`) | `zfs list` | +| `zfs_snapshots` | The snapshots of one dataset (`dataset`, optional), oldest first: creation time, used (the space only that snapshot holds) and referenced | `zfs list -t snapshot` | +| `zfs_snapshot` | Takes `dataset@name` (see below) | `zfs snapshot` | +| `zfs_pools` | Each pool's state with status's explanation and action, space, capacity and fragmentation %, device error count, data errors and last scan | `zpool list`, `zpool status -x` | +| `zfs_space` | Each pool's space, then every dataset, biggest first: usedbydataset, usedbysnapshots, usedbychildren and usedbyrefreservation, quotas and reservations where set, compression, logical size against physical size, and the savings | `zfs list -o space`, `zpool list -v` | +| `zfs_pool_layout` | Vdevs and disks by the names the pool knows them by, with state, read, write and checksum errors and status's note; log, cache, special and dedup devices; spares | `zpool status`, `zpool status -v` | +| `zfs_scrub` | The last scan (scrub or resilver): when, how long, what it repaired, the errors, or its progress while it runs; the days since the last finished scrub, with a warning past 35. With `history: true`, it also reads the pool's history (as root) for the scrub and resilver that status no longer shows | `zpool history -i` | +| `zfs_io` | A sample of `seconds` (default 3, from 1 to 10): operations and bytes per second per pool, vdev and disk | `zpool iostat -v` | +| `zfs_arc` | ARC size, target, minimum and maximum, hit ratio since boot, demand hit ratio, and L2ARC where present | `arcstat`, `arc_summary` | +| `zfs_events` | Recent events, newest first, as root. By default it shows only errors and changes of state, and `all: true` shows every event (`limit`, default 50) | `zpool events -v` | +| `zfs_properties` | Every property of one `dataset` with its source, the usual ones in lines first. The value of `keylocation` is withheld, and ZFS never gives a key itself | `zfs get all` | +| `zfs_health` | The health check (below) | | + +## The snapshot + +`zfs_snapshot {dataset, name, why, recursive?}` refuses all of these before anything runs: + +- a dataset that `zfs list` does not show on this machine; +- a name with anything but letters, digits, `.`, `_` and `-`; +- a whole name longer than ZFS's 255 characters; +- a snapshot that already exists (with `recursive`, on any child too); +- a missing `why`, or a `why` longer than one line. + +It then runs `sudo -n zfs snapshot -- @`, with `-r` only when `recursive` is true. It +answers from `zfs list` afterwards, not from what it ran: the snapshot made and its creation time, and any +other snapshot that came or went during the call (another schedule's, for example). A snapshot the list +does not show is an error, whatever the command said. + +The act is recorded as the event `snapshot.taken` (the dataset, the name, why, whether it was done, and +when), and ZFS keeps the command in the pool's own history. An event the bus does not accept is said in +the answer as NOT RECORDED. + +## Health + +The module runs two long-running processes of its own binary, and neither writes a file or changes +anything: + +- **The watch** (`zfs-tools watch`) carries the module's health check, `zfs_health` (novox/hq ADR 0240). + Each minute it reads the pools and writes a line in its journal whenever their health changes. +- **The event follower** (`zfs-tools events`) runs as root and follows `zpool events -f`. It writes each + error and change of state to its journal as ZFS emits it. ZFS keeps its events in memory only, so they + are lost at a reboot, and the journal keeps them past one. The node-engine judges it by its unit. That + is the check the module does not run itself, which a tool check needs beside it (ADR 0240 rule 2). + +`zfs_health` is **unhealthy** when: + +- a pool is not ONLINE (the answer gives status's reason and action); +- a device has read, write or checksum errors, or is not online; +- a pool reports data errors; +- a pool is more than 90 % full; +- no pool is imported; +- ZFS is not on the machine. + +These are **warnings**, which leave the module healthy: + +- a pool is more than 80 % full; +- a pool's last scrub finished more than 35 days ago, or the pool was never scrubbed. + +The health check judges only the last scan that status shows. If that scan was a resilver, the last +scrub's age is in `zfs_scrub` with `history: true`. + +## Tests + +`go test ./...` runs against OpenZFS 2.4 output in `cmd/zfs-tools/testdata`. Each file has the exact form +the command prints, with documentation names in place of the real pools and disks. The tests cover: + +- parsing every command's output; +- the snapshot's argument list, every refusal (none runs a command), the read-back, a snapshot the list + does not show, a refused sudo, an event the bus does not accept, and another snapshot taken meanwhile; +- the health verdicts; +- a machine without ZFS; +- the manifest against the tools the bundle serves. diff --git a/modules/zfs/cmd/zfs-tools/datasets.go b/modules/zfs/cmd/zfs-tools/datasets.go new file mode 100644 index 00000000..3c4e7597 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/datasets.go @@ -0,0 +1,423 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// DatasetColumns are the properties `zfs list` is asked for, in order: every one in bytes under -p but +// the name, the type, the mountpoint, sharenfs, compression and the compression ratio. +var DatasetColumns = []string{ + "name", "type", "mountpoint", "used", "available", "referenced", "compressratio", "sharenfs", + "usedbysnapshots", "usedbydataset", "usedbychildren", "usedbyrefreservation", + "quota", "refquota", "reservation", "refreservation", "logicalused", "logicalreferenced", "compression", +} + +// SnapshotColumns are the properties of a snapshot `zfs list -t snapshot` is asked for. +var SnapshotColumns = []string{"name", "creation", "used", "referenced"} + +// DatasetsArgv is the one command that lists every filesystem and volume. +func DatasetsArgv() []string { + return []string{"list", "-Hp", "-t", "filesystem,volume", "-o", strings.Join(DatasetColumns, ",")} +} + +// SnapshotsArgv is the one command that lists every snapshot, oldest first. +func SnapshotsArgv() []string { + return []string{"list", "-Hp", "-t", "snapshot", "-s", "creation", "-o", strings.Join(SnapshotColumns, ",")} +} + +// Dataset is one filesystem or volume, its sizes in bytes. +type Dataset struct { + Name string `json:"name"` + Type string `json:"type"` + Pool string `json:"pool"` + Mountpoint string `json:"mountpoint,omitempty"` + Used uint64 `json:"used"` + Available uint64 `json:"available"` + Referenced uint64 `json:"referenced"` + // CompressRatio is the ratio of the logical to the physical size of what it references, 1.00 for none. + CompressRatio float64 `json:"compressratio"` + Compression string `json:"compression,omitempty"` + ShareNFS string `json:"sharenfs,omitempty"` + + UsedBySnapshots uint64 `json:"usedbysnapshots"` + UsedByDataset uint64 `json:"usedbydataset"` + UsedByChildren uint64 `json:"usedbychildren"` + UsedByRefreservation uint64 `json:"usedbyrefreservation"` + // Quota, RefQuota, Reservation and RefReservation are nil when none is set. + Quota *uint64 `json:"quota,omitempty"` + RefQuota *uint64 `json:"refquota,omitempty"` + Reservation *uint64 `json:"reservation,omitempty"` + RefReservation *uint64 `json:"refreservation,omitempty"` + LogicalUsed uint64 `json:"logicalused"` + LogicalReferenced uint64 `json:"logicalreferenced"` +} + +// Snapshot is one snapshot. +type Snapshot struct { + Name string `json:"name"` + Dataset string `json:"dataset"` + Snap string `json:"snapshot"` + Created time.Time `json:"created"` + // Used is the space only this snapshot holds: what destroying it alone would free. + Used uint64 `json:"used"` + Referenced uint64 `json:"referenced"` +} + +// ParseDatasets reads `zfs list -Hp -o `. A line that does not have every column is +// refused by name, never skipped silently. +func ParseDatasets(out string) ([]Dataset, error) { + var ds []Dataset + for _, line := range lines(out) { + f := strings.Split(line, "\t") + if len(f) != len(DatasetColumns) { + return nil, fmt.Errorf("zfs list answered a line with %d columns, not %d: %q", len(f), len(DatasetColumns), line) + } + col := func(name string) string { return f[index(DatasetColumns, name)] } + d := Dataset{Name: col("name"), Type: col("type"), Pool: poolOf(col("name")), Compression: dash(col("compression")), + Mountpoint: dash(col("mountpoint")), ShareNFS: dash(col("sharenfs"))} + var err error + num := func(name string) uint64 { + v, e := number(col(name)) + if e != nil && err == nil { + err = fmt.Errorf("zfs list: %s of %s: %v", name, d.Name, e) + } + return v + } + d.Used, d.Available, d.Referenced = num("used"), num("available"), num("referenced") + d.UsedBySnapshots, d.UsedByDataset = num("usedbysnapshots"), num("usedbydataset") + d.UsedByChildren, d.UsedByRefreservation = num("usedbychildren"), num("usedbyrefreservation") + d.LogicalUsed, d.LogicalReferenced = num("logicalused"), num("logicalreferenced") + d.Quota, d.RefQuota = optional(col("quota")), optional(col("refquota")) + d.Reservation, d.RefReservation = optional(col("reservation")), optional(col("refreservation")) + if err != nil { + return nil, err + } + cr, rerr := ratio(col("compressratio")) + if rerr != nil { + return nil, fmt.Errorf("zfs list: compressratio of %s: %v", d.Name, rerr) + } + d.CompressRatio = cr + ds = append(ds, d) + } + return ds, nil +} + +// ParseSnapshots reads `zfs list -Hp -t snapshot -o `. +func ParseSnapshots(out string) ([]Snapshot, error) { + var ss []Snapshot + for _, line := range lines(out) { + f := strings.Split(line, "\t") + if len(f) != len(SnapshotColumns) { + return nil, fmt.Errorf("zfs list -t snapshot answered a line with %d columns, not %d: %q", len(f), len(SnapshotColumns), line) + } + at := strings.IndexByte(f[0], '@') + if at < 1 { + return nil, fmt.Errorf("zfs list -t snapshot answered %q, which is not a snapshot's name", f[0]) + } + s := Snapshot{Name: f[0], Dataset: f[0][:at], Snap: f[0][at+1:]} + created, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("zfs list: creation of %s is %q, not seconds", s.Name, f[1]) + } + s.Created = time.Unix(created, 0) + if s.Used, err = number(f[2]); err != nil { + return nil, fmt.Errorf("zfs list: used of %s: %v", s.Name, err) + } + if s.Referenced, err = number(f[3]); err != nil { + return nil, fmt.Errorf("zfs list: referenced of %s: %v", s.Name, err) + } + ss = append(ss, s) + } + return ss, nil +} + +// Datasets are every filesystem and volume on the machine. +func (m *Machine) Datasets(ctx context.Context) ([]Dataset, error) { + out, err := m.ask(ctx, "zfs", DatasetsArgv()...) + if err != nil { + return nil, err + } + return ParseDatasets(out) +} + +// Snapshots are every snapshot on the machine, oldest first. +func (m *Machine) Snapshots(ctx context.Context) ([]Snapshot, error) { + out, err := m.ask(ctx, "zfs", SnapshotsArgv()...) + if err != nil { + return nil, err + } + return ParseSnapshots(out) +} + +// DatasetRow is one dataset as zfs_datasets answers it. +type DatasetRow struct { + Name string `json:"name"` + Type string `json:"type"` + Mountpoint string `json:"mountpoint,omitempty"` + Used uint64 `json:"used_bytes"` + Available uint64 `json:"available_bytes"` + Referenced uint64 `json:"referenced_bytes"` + CompressRatio float64 `json:"compressratio"` + ShareNFS string `json:"sharenfs,omitempty"` + Snapshots int `json:"snapshots"` + // UsedBySnapshots is the space the snapshots hold together: what destroying all of them would free. + UsedBySnapshots uint64 `json:"usedbysnapshots_bytes"` + Said string `json:"said"` +} + +// DatasetsAnswer is what zfs_datasets answers. +type DatasetsAnswer struct { + Applicable bool `json:"applicable"` + Datasets []DatasetRow `json:"datasets"` + Count int `json:"count"` + Said []string `json:"said"` +} + +// DatasetsNow lists every dataset with its space and how many snapshots it has. +func (m *Machine) DatasetsNow(ctx context.Context) (any, error) { + ds, err := m.Datasets(ctx) + if err != nil { + return notHere(err) + } + ss, err := m.Snapshots(ctx) + if err != nil { + return nil, err + } + count := map[string]int{} + for _, s := range ss { + count[s.Dataset]++ + } + a := DatasetsAnswer{Applicable: true, Datasets: []DatasetRow{}, Said: []string{}} + for _, d := range ds { + r := DatasetRow{Name: d.Name, Type: d.Type, Mountpoint: d.Mountpoint, Used: d.Used, Available: d.Available, + Referenced: d.Referenced, CompressRatio: d.CompressRatio, ShareNFS: d.ShareNFS, Snapshots: count[d.Name], + UsedBySnapshots: d.UsedBySnapshots} + where := d.Mountpoint + if where == "" { + where = d.Type + } + r.Said = fmt.Sprintf("%s (%s): %s used, %s free, %d snapshots holding %s, compression %.2fx", + d.Name, where, bytesSaid(d.Used), bytesSaid(d.Available), r.Snapshots, bytesSaid(d.UsedBySnapshots), d.CompressRatio) + if d.ShareNFS != "" && d.ShareNFS != "off" { + r.Said += ", sharenfs " + d.ShareNFS + } + a.Datasets = append(a.Datasets, r) + a.Said = append(a.Said, r.Said) + } + a.Count = len(a.Datasets) + return a, nil +} + +// SnapshotRow is one snapshot as zfs_snapshots answers it. +type SnapshotRow struct { + Name string `json:"name"` + Dataset string `json:"dataset"` + Created string `json:"created"` + Used uint64 `json:"used_bytes"` + Referenced uint64 `json:"referenced_bytes"` +} + +// SnapshotsAnswer is what zfs_snapshots answers. +type SnapshotsAnswer struct { + Applicable bool `json:"applicable"` + Dataset string `json:"dataset,omitempty"` + Snapshots []SnapshotRow `json:"snapshots"` + Count int `json:"count"` + // Unique is the sum of the snapshots' own space: less than they hold together, which a dataset's + // usedbysnapshots says. + Unique uint64 `json:"used_bytes_sum"` + Said string `json:"said"` +} + +// SnapshotsOf lists the snapshots of one dataset (exactly that one, not its children), or of all. +func (m *Machine) SnapshotsOf(ctx context.Context, dataset string) (any, error) { + if dataset != "" { + ds, err := m.Datasets(ctx) + if err != nil { + return notHere(err) + } + if find(ds, dataset) == nil { + return nil, fmt.Errorf("there is no dataset %q on this machine; zfs_datasets lists them", dataset) + } + } + ss, err := m.Snapshots(ctx) + if err != nil { + return notHere(err) + } + a := SnapshotsAnswer{Applicable: true, Dataset: dataset, Snapshots: []SnapshotRow{}} + for _, s := range ss { + if dataset != "" && s.Dataset != dataset { + continue + } + a.Snapshots = append(a.Snapshots, SnapshotRow{Name: s.Name, Dataset: s.Dataset, Created: s.Created.Local().Format(time.RFC3339), + Used: s.Used, Referenced: s.Referenced}) + a.Unique += s.Used + } + a.Count = len(a.Snapshots) + of := "on this machine" + if dataset != "" { + of = "of " + dataset + } + a.Said = fmt.Sprintf("%d snapshots %s, oldest first; each one's own space adds up to %s.", a.Count, of, bytesSaid(a.Unique)) + return a, nil +} + +// SpaceRow is one dataset's space as zfs_space answers it. +type SpaceRow struct { + Name string `json:"name"` + Used uint64 `json:"used_bytes"` + Available uint64 `json:"available_bytes"` + UsedByDataset uint64 `json:"usedbydataset_bytes"` + UsedBySnapshots uint64 `json:"usedbysnapshots_bytes"` + UsedByChildren uint64 `json:"usedbychildren_bytes"` + UsedByRefreservation uint64 `json:"usedbyrefreservation_bytes"` + Quota *uint64 `json:"quota_bytes,omitempty"` + RefQuota *uint64 `json:"refquota_bytes,omitempty"` + Reservation *uint64 `json:"reservation_bytes,omitempty"` + RefReservation *uint64 `json:"refreservation_bytes,omitempty"` + CompressRatio float64 `json:"compressratio"` + Compression string `json:"compression,omitempty"` + LogicalUsed uint64 `json:"logicalused_bytes"` + // Saved is what compression saves: the logical size less the physical, never below zero. + Saved uint64 `json:"saved_bytes"` + Said string `json:"said"` +} + +// SpaceAnswer is what zfs_space answers. +type SpaceAnswer struct { + Applicable bool `json:"applicable"` + Pools []PoolRow `json:"pools"` + Datasets []SpaceRow `json:"datasets"` + Said []string `json:"said"` +} + +// Space is every pool's space and every dataset's, biggest first, with what uses it. +func (m *Machine) Space(ctx context.Context) (any, error) { + pools, err := m.Pools(ctx) + if err != nil { + return notHere(err) + } + ds, err := m.Datasets(ctx) + if err != nil { + return nil, err + } + a := SpaceAnswer{Applicable: true, Pools: []PoolRow{}, Datasets: []SpaceRow{}, Said: []string{}} + for _, p := range pools { + r := p.Row() + a.Pools = append(a.Pools, r) + a.Said = append(a.Said, r.Said) + } + sort.SliceStable(ds, func(i, j int) bool { return ds[i].Used > ds[j].Used }) + for _, d := range ds { + r := SpaceRow{Name: d.Name, Used: d.Used, Available: d.Available, UsedByDataset: d.UsedByDataset, + UsedBySnapshots: d.UsedBySnapshots, UsedByChildren: d.UsedByChildren, UsedByRefreservation: d.UsedByRefreservation, + Quota: d.Quota, RefQuota: d.RefQuota, Reservation: d.Reservation, RefReservation: d.RefReservation, + CompressRatio: d.CompressRatio, Compression: d.Compression, LogicalUsed: d.LogicalUsed} + if d.LogicalUsed > d.Used { + r.Saved = d.LogicalUsed - d.Used + } + r.Said = fmt.Sprintf("%s: %s used (itself %s, snapshots %s, children %s, refreservation %s), %s free; "+ + "%s logical, %.2fx, %s saved", d.Name, bytesSaid(d.Used), bytesSaid(d.UsedByDataset), bytesSaid(d.UsedBySnapshots), + bytesSaid(d.UsedByChildren), bytesSaid(d.UsedByRefreservation), bytesSaid(d.Available), bytesSaid(d.LogicalUsed), + d.CompressRatio, bytesSaid(r.Saved)) + for _, l := range []struct { + name string + v *uint64 + }{{"quota", d.Quota}, {"refquota", d.RefQuota}, {"reservation", d.Reservation}, {"refreservation", d.RefReservation}} { + if l.v != nil { + r.Said += fmt.Sprintf(", %s %s", l.name, bytesSaid(*l.v)) + } + } + a.Datasets = append(a.Datasets, r) + a.Said = append(a.Said, r.Said) + } + return a, nil +} + +// notHere answers a machine without ZFS as not applicable, and any other failure as itself. +func notHere(err error) (any, error) { + if a, ok := err.(errAbsent); ok { + return NotApplicable{Applicable: false, Said: a.why}, nil + } + return nil, err +} + +func find(ds []Dataset, name string) *Dataset { + for i := range ds { + if ds[i].Name == name { + return &ds[i] + } + } + return nil +} + +func poolOf(name string) string { + if i := strings.IndexAny(name, "/@"); i >= 0 { + return name[:i] + } + return name +} + +func lines(out string) []string { + var ls []string + for _, l := range strings.Split(out, "\n") { + if strings.TrimSpace(l) != "" { + ls = append(ls, strings.TrimRight(l, "\r")) + } + } + return ls +} + +func index(cols []string, name string) int { + for i, c := range cols { + if c == name { + return i + } + } + panic("no column " + name) +} + +// dash is a property's value, "" for ZFS's "-" (not applicable). +func dash(s string) string { + if s == "-" { + return "" + } + return s +} + +// number is a size under -p: whole bytes. "-" (not applicable to this type) is 0. +func number(s string) (uint64, error) { + if s == "-" || s == "" { + return 0, nil + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, fmt.Errorf("%q is not a whole number of bytes", s) + } + return v, nil +} + +// optional is a quota or a reservation: nil when none is set (0 under -p, or "-" or "none"). +func optional(s string) *uint64 { + if s == "-" || s == "none" || s == "" { + return nil + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil || v == 0 { + return nil + } + return &v +} + +// ratio is a compression or dedup ratio, "1.52" under -p or "1.52x" without. +func ratio(s string) (float64, error) { + if s == "-" || s == "" { + return 1, nil + } + return strconv.ParseFloat(strings.TrimSuffix(s, "x"), 64) +} diff --git a/modules/zfs/cmd/zfs-tools/events.go b/modules/zfs/cmd/zfs-tools/events.go new file mode 100644 index 00000000..29d04362 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/events.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" +) + +// EventsArgv is ZFS's recent events with their details. ZFS keeps them in memory since the module loaded, +// a bounded number, and gives them to root only. +func EventsArgv() []string { return []string{"events", "-v"} } + +// Event is one ZFS event. +type Event struct { + Time string `json:"time"` + Class string `json:"class"` + // Details are the event's fields that say what and where: pool, vdev, state, error. + Details map[string]string `json:"details,omitempty"` +} + +// eventKeys are the details an event is answered with; the rest are internal identifiers and checksums. +var eventKeys = map[string]bool{ + "pool": true, "pool_state": true, "vdev_path": true, "vdev_type": true, "vdev_state": true, "vdev_laststate": true, + "vdev_read_errors": true, "vdev_write_errors": true, "vdev_cksum_errors": true, "zio_err": true, "zio_objset": true, + "zio_object": true, "history_internal_name": true, "history_internal_str": true, "vdev_ashift": true, +} + +// vdevStates are ZFS's vdev states as numbers, as events give them. +var vdevStates = map[string]string{ + "0x0": "UNKNOWN", "0x1": "CLOSED", "0x2": "OFFLINE", "0x3": "REMOVED", "0x4": "CANT_OPEN", "0x5": "FAULTED", + "0x6": "DEGRADED", "0x7": "ONLINE", +} + +// ParseEvents reads `zpool events -v`: a header line, then for each event a line of its time and class and +// its fields indented under it, `name = value`. +func ParseEvents(out string) []Event { + var evs []Event + for _, line := range strings.Split(out, "\n") { + line = strings.TrimRight(line, "\r") + if strings.TrimSpace(line) == "" { + continue + } + if line[0] != ' ' && line[0] != '\t' { + f := strings.Fields(line) + if len(f) < 2 || (f[0] == "TIME" && f[len(f)-1] == "CLASS") { + continue + } + evs = append(evs, Event{Time: strings.Join(f[:len(f)-1], " "), Class: f[len(f)-1]}) + continue + } + if len(evs) == 0 { + continue + } + k, v, ok := strings.Cut(strings.TrimSpace(line), " = ") + if !ok || !eventKeys[k] { + continue + } + v = strings.Trim(strings.TrimSpace(v), `"`) + if k == "vdev_state" || k == "vdev_laststate" { + if s, known := vdevStates[v]; known { + v = s + } + } + e := &evs[len(evs)-1] + if e.Details == nil { + e.Details = map[string]string{} + } + e.Details[k] = v + } + return evs +} + +// Notable says whether an event is an error or a change of state: every ereport (an I/O, checksum or +// device error), every resource event (a device's state changed) and ZFS's own events but the history +// it writes for every command and the configuration it syncs. +func Notable(class string) bool { + switch { + case strings.HasPrefix(class, "ereport."), strings.HasPrefix(class, "resource."): + return true + case class == "sysevent.fs.zfs.history_event", class == "sysevent.fs.zfs.config_sync": + return false + case strings.HasPrefix(class, "sysevent.fs.zfs."): + return true + } + return false +} + +// EventsAnswer is what zfs_events answers. +type EventsAnswer struct { + Applicable bool `json:"applicable"` + Events []Event `json:"events"` + Shown int `json:"shown"` + Kept int `json:"kept"` + Said []string `json:"said"` +} + +// Events are ZFS's recent errors and changes of state, newest first; with all, every event. +func (m *Machine) Events(ctx context.Context, all bool, limit int) (any, error) { + r, err := m.AsRoot(ctx, "zpool", EventsArgv()...) + if err != nil { + return notHere(err) + } + evs := ParseEvents(r.Out) + a := EventsAnswer{Applicable: true, Events: []Event{}, Kept: len(evs), Said: []string{}} + for i := len(evs) - 1; i >= 0 && len(a.Events) < limit; i-- { + if all || Notable(evs[i].Class) { + a.Events = append(a.Events, evs[i]) + } + } + a.Shown = len(a.Events) + for _, e := range a.Events { + line := e.Time + " " + strings.TrimPrefix(strings.TrimPrefix(e.Class, "sysevent.fs.zfs."), "ereport.fs.zfs.") + keys := make([]string, 0, len(e.Details)) + for k := range e.Details { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + line += fmt.Sprintf(" %s=%s", k, e.Details[k]) + } + a.Said = append(a.Said, line) + } + which := "errors and changes of state" + if all { + which = "events" + } + a.Said = append([]string{fmt.Sprintf("%d %s shown, newest first, of the %d events ZFS keeps since its module loaded", a.Shown, which, a.Kept)}, a.Said...) + return a, nil +} diff --git a/modules/zfs/cmd/zfs-tools/follow.go b/modules/zfs/cmd/zfs-tools/follow.go new file mode 100644 index 00000000..0fae9297 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/follow.go @@ -0,0 +1,104 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "os/exec" + "sort" + "strings" + "time" +) + +// The module's second long-running process (`zfs-tools events`), run by the node-engine as root. ZFS keeps +// its events in memory only, so a reboot or a reload of its kernel module loses them; the follower writes +// each error and change of state to its journal as ZFS emits it, where it outlives both. It writes no file +// and changes nothing. It is judged by its unit (novox/hq ADR 0240), the check beside zfs_health that the +// module does not run itself. + +// FollowArgv follows ZFS's events as they come, with their details. +func FollowArgv() []string { return []string{"events", "-f", "-v"} } + +// FollowRetry is how long the follower waits before following again after zpool exits. +const FollowRetry = 30 * time.Second + +// eventTime reads an event's time as `zpool events` gives it: "Oct 8 2026 21:01:55.889900112". +func eventTime(s string) (time.Time, bool) { + t, err := time.ParseInLocation("Jan 2 2006 15:04:05.999999999", strings.Join(strings.Fields(s), " "), time.Local) + return t, err == nil +} + +// EventLine is an event in one journal line. +func EventLine(e Event) string { + line := e.Time + " " + e.Class + keys := make([]string, 0, len(e.Details)) + for k := range e.Details { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + line += fmt.Sprintf(" %s=%s", k, e.Details[k]) + } + return line +} + +// Follow reads `zpool events -f -v` and writes each notable event at or after since in a line. `-f` first +// gives every event ZFS still keeps; those from before since were written by an earlier follower, or are +// in zfs_events, and are not written again. +func Follow(r io.Reader, out io.Writer, since time.Time) error { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + var block []string + flush := func() { + for _, e := range ParseEvents(strings.Join(block, "\n")) { + if !Notable(e.Class) { + continue + } + if t, ok := eventTime(e.Time); ok && t.Before(since) { + continue + } + fmt.Fprintln(out, EventLine(e)) + } + block = block[:0] + } + for sc.Scan() { + l := sc.Text() + if strings.TrimSpace(l) == "" { + flush() + continue + } + block = append(block, l) + } + flush() + return sc.Err() +} + +// Follower runs zpool events -f until the context ends, again after FollowRetry whenever it exits. +func Follower(ctx context.Context, out io.Writer) { + since := time.Now() + for ctx.Err() == nil { + cmd := exec.CommandContext(ctx, "zpool", FollowArgv()...) + stdout, err := cmd.StdoutPipe() + if err == nil { + err = cmd.Start() + } + if err == nil { + err = Follow(stdout, out, since) + if werr := cmd.Wait(); err == nil { + err = werr + } + } + if ctx.Err() != nil { + return + } + fmt.Fprintf(out, "zpool events -f stopped (%v); following again in %s\n", err, FollowRetry) + // What comes while it waits is still in ZFS's own list, and the next follow writes it from here. + since = time.Now() + select { + case <-ctx.Done(): + return + case <-time.After(FollowRetry): + } + } +} diff --git a/modules/zfs/cmd/zfs-tools/health.go b/modules/zfs/cmd/zfs-tools/health.go new file mode 100644 index 00000000..633eff86 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/health.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" +) + +// The lines a pool's capacity is judged against: past CapacityWarn a warning, past CapacityFail +// unhealthy. ZFS slows as a pool fills, and a full pool can refuse even a deletion. +const ( + CapacityWarn = 80 + CapacityFail = 90 +) + +// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why, one +// sentence per finding; warnings do not make it unhealthy. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why"` + Found []string `json:"found,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// Judge judges the pools from what zpool list and zpool status said. +func Judge(pools []Pool, status []PoolStatus, now time.Time) Health { + var found, warn []string + if len(pools) == 0 { + found = append(found, "ZFS is on this machine and no pool is imported.") + } + for _, p := range pools { + s := statusOf(status, p.Name) + state := p.Health + if s != nil && s.State != "" { + state = s.State + } + if state != "ONLINE" { + line := fmt.Sprintf("Pool %s is %s", p.Name, state) + if s != nil && s.Status != "" { + line += ": " + strings.TrimSuffix(s.Status, ".") + } + if s != nil && s.Action != "" { + line += ". Action: " + strings.TrimSuffix(s.Action, ".") + } + found = append(found, line+".") + } + switch { + case p.Capacity > CapacityFail: + found = append(found, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityFail, bytesSaid(p.Free))) + case p.Capacity > CapacityWarn: + warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityWarn, bytesSaid(p.Free))) + } + if s == nil { + found = append(found, fmt.Sprintf("zpool status said nothing of pool %s.", p.Name)) + continue + } + walk(s.Root, func(d *Device, _ int) { + if d.Kind == "section" { + return + } + if d.Errors() > 0 { + found = append(found, fmt.Sprintf("%s %s in pool %s has errors: read %d, write %d, checksum %d.", d.Kind, d.Name, p.Name, d.Read, d.Write, d.Checksum)) + } + if d.Kind != "pool" && d.State != "" && d.State != "ONLINE" && d.State != "AVAIL" && d.State != "INUSE" { + // A device not online already makes its pool DEGRADED or worse; said here by name. + found = append(found, fmt.Sprintf("%s %s in pool %s is %s.", d.Kind, d.Name, p.Name, d.State)) + } + }, 0) + if e := strings.TrimSpace(s.Errors); e != "" && e != "No known data errors" { + found = append(found, fmt.Sprintf("Pool %s: %s.", p.Name, strings.TrimSuffix(e, "."))) + } + if ended := LastScrubEnded(s.Scan); ended != nil && now.Sub(*ended) > ScrubWarnAfter { + warn = append(warn, fmt.Sprintf("Pool %s was last scrubbed %.0f days ago, more than %d.", p.Name, now.Sub(*ended).Hours()/24, int(ScrubWarnAfter.Hours()/24))) + } else if s.Scan.State == "none" { + warn = append(warn, fmt.Sprintf("Pool %s was never scrubbed.", p.Name)) + } + } + if len(found) > 0 { + return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn} + } + var parts []string + for _, p := range pools { + parts = append(parts, fmt.Sprintf("%s ONLINE, %d %% full", p.Name, p.Capacity)) + } + why := "Every pool is online with no errors: " + strings.Join(parts, "; ") + "." + if len(warn) > 0 { + why += " Warnings: " + strings.Join(warn, " ") + } + return Health{Healthy: true, Why: why, Warnings: warn} +} + +// Health reads the pools and judges them. A machine without ZFS is unhealthy, loudly: the module was put +// where it does not apply. +func (m *Machine) Health(ctx context.Context) Health { + pools, err := m.Pools(ctx) + if err != nil { + if a, ok := err.(errAbsent); ok { + return Health{Healthy: false, Why: sentence(a.why) + " Unassign the module from this machine.", Found: []string{a.why}} + } + return Health{Healthy: false, Why: sentence("zpool list did not answer: " + err.Error()), Found: []string{err.Error()}} + } + status, err := m.Status(ctx) + if err != nil { + return Health{Healthy: false, Why: sentence("zpool status did not answer: " + err.Error()), Found: []string{err.Error()}} + } + return Judge(pools, status, m.Now()) +} + +func sentence(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return s + } + s = strings.ToUpper(s[:1]) + s[1:] + if !strings.HasSuffix(s, ".") { + s += "." + } + return s +} diff --git a/modules/zfs/cmd/zfs-tools/health_test.go b/modules/zfs/cmd/zfs-tools/health_test.go new file mode 100644 index 00000000..ba80be9d --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/health_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" + "time" +) + +func TestOnlinePoolsWithoutErrorsAreHealthyAndAnOldScrubIsOnlyAWarning(t *testing.T) { + h := withZFS(t).machine().Health(context.Background()) + if !h.Healthy || len(h.Found) != 0 { + t.Fatalf("health: %+v", h) + } + if len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "Pool rpool was last scrubbed 89 days ago, more than 35") { + t.Errorf("rpool's July scrub is a warning; tank's September one is not: %v", h.Warnings) + } + if !strings.Contains(h.Why, "tank ONLINE, 60 % full") { + t.Errorf("why: %s", h.Why) + } +} + +func TestADegradedFullPoolIsUnhealthyWithEveryReason(t *testing.T) { + f := newFake(t) + f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list-full.txt")) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + h := f.machine().Health(context.Background()) + if h.Healthy { + t.Fatalf("healthy: %+v", h) + } + for _, want := range []string{ + "Pool tank is DEGRADED: One or more devices is currently being resilvered", + "Action: Wait for the resilver to complete", + "Pool tank is 91 % full, past 90 %", + "device ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank has errors: read 12, write 107, checksum 3", + "device ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank is FAULTED", + "Pool tank: 2 data errors", + } { + if !strings.Contains(h.Why, want) { + t.Errorf("why lacks %q: %s", want, h.Why) + } + } + if strings.Contains(h.Why, "D7JA0009 in pool tank is") { + t.Errorf("a spare in use is not a finding: %s", h.Why) + } +} + +func TestCapacityPastEightyIsAWarningAndPastNinetyUnhealthy(t *testing.T) { + st, _ := ParseStatus(fixture(t, "zpool-status-scrubbing.txt")) + now := newFake(t).now + p := Pool{Name: "tank", Capacity: 85, Health: "ONLINE"} + if h := Judge([]Pool{p}, st, now); !h.Healthy || len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "85 % full, past 80 %") { + t.Errorf("85 %%: %+v", h) + } + p.Capacity = 80 + if h := Judge([]Pool{p}, st, now); !h.Healthy || len(h.Warnings) != 0 { + t.Errorf("80 %% is not past 80: %+v", h) + } + p.Capacity = 92 + if h := Judge([]Pool{p}, st, now); h.Healthy { + t.Errorf("92 %%: %+v", h) + } +} + +func TestFaultedAndNoPoolAreUnhealthy(t *testing.T) { + now := newFake(t).now + if h := Judge(nil, nil, now); h.Healthy || !strings.Contains(h.Why, "no pool is imported") { + t.Errorf("no pool: %+v", h) + } + st := []PoolStatus{{Name: "tank", State: "FAULTED", Status: "One or more devices could not be opened.", Errors: "No known data errors", + Root: &Device{Name: "tank", Kind: "pool", State: "FAULTED"}}} + if h := Judge([]Pool{{Name: "tank", Health: "FAULTED"}}, st, now); h.Healthy || !strings.Contains(h.Why, "Pool tank is FAULTED: One or more devices could not be opened") { + t.Errorf("faulted: %+v", h) + } +} + +func TestTheWatchSaysOnlyChanges(t *testing.T) { + f := withZFS(t) + var out bytes.Buffer + w := &Watch{M: f.machine(), Out: &out} + w.Look(context.Background()) + w.Look(context.Background()) + if n := strings.Count(out.String(), "\n"); n != 1 || !strings.Contains(out.String(), "healthy: Every pool is online") { + t.Errorf("one line for one state: %q", out.String()) + } + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + w.Look(context.Background()) + if !strings.Contains(out.String(), "UNHEALTHY: ") { + t.Errorf("the change is said: %q", out.String()) + } +} + +func TestTheFollowerWritesEachNotableEventFromItsStartOnce(t *testing.T) { + var out bytes.Buffer + since := time.Date(2026, 10, 8, 21, 0, 0, 0, time.Local) + if err := Follow(strings.NewReader(fixture(t, "zpool-events.txt")), &out, since); err != nil { + t.Fatal(err) + } + got := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(got) != 3 || !strings.Contains(got[0], "ereport.fs.zfs.io") || !strings.Contains(got[1], "vdev_state=FAULTED") || + !strings.Contains(got[2], "resilver_start") { + t.Errorf("lines: %q", got) + } + out.Reset() + Follow(strings.NewReader(fixture(t, "zpool-events.txt")), &out, time.Date(2026, 10, 8, 21, 2, 10, 0, time.Local)) + if got := strings.TrimSpace(out.String()); strings.Count(got, "\n") != 0 || !strings.Contains(got, "resilver_start") { + t.Errorf("only what came after since: %q", got) + } + if strings.Join(FollowArgv(), " ") != "events -f -v" { + t.Errorf("argv: %v", FollowArgv()) + } +} diff --git a/modules/zfs/cmd/zfs-tools/helpers_test.go b/modules/zfs/cmd/zfs-tools/helpers_test.go new file mode 100644 index 00000000..1a4c4f28 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/helpers_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The fixtures in testdata are the output of OpenZFS 2.4 (the version the home server runs) in the exact +// form each command prints it — tab-separated under -H, exact numbers under -p — with the pool, disk and +// address names replaced by documentation ones. + +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// fake is a machine for a test: a table of command lines and their answers, every call kept. +type fake struct { + t *testing.T + answers map[string]Ran + // after runs once a command line was answered: a snapshot's command changes what the list answers. + after map[string]func(*fake) + calls []string + emitted []map[string]any + emitErr error + files map[string]string + now time.Time + uid int +} + +func newFake(t *testing.T) *fake { + return &fake{t: t, answers: map[string]Ran{}, after: map[string]func(*fake){}, files: map[string]string{}, + now: time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local), uid: 1000} +} + +func (f *fake) on(line string, out string) { f.answers[line] = Ran{Out: out} } + +func (f *fake) machine() *Machine { + return &Machine{ + Run: func(_ context.Context, name string, args ...string) Ran { + line := strings.TrimSpace(name + " " + strings.Join(args, " ")) + f.calls = append(f.calls, line) + r, ok := f.answers[line] + if !ok && name == "sudo" && len(args) > 1 { + // sudo is there; the command it was asked for is not. + r = Ran{Code: 1, Err: "sudo: " + args[1] + ": command not found"} + } else if !ok { + r = Ran{Code: -1, Err: "exec: \"" + name + "\": executable file not found in $PATH"} + } + if then, ok := f.after[line]; ok { + then(f) + } + return r + }, + UID: f.uid, + Now: func() time.Time { return f.now }, + ReadFile: func(p string) ([]byte, error) { + s, ok := f.files[p] + if !ok { + return nil, os.ErrNotExist + } + return []byte(s), nil + }, + Emit: func(eventType string, body any) error { + if f.emitErr != nil { + return f.emitErr + } + b := body.(map[string]any) + b["_type"] = eventType + f.emitted = append(f.emitted, b) + return nil + }, + } +} + +func line(name string, argv []string) string { return name + " " + strings.Join(argv, " ") } + +// withZFS answers the read commands from the healthy fixtures. +func withZFS(t *testing.T) *fake { + f := newFake(t) + f.on(line("zfs", DatasetsArgv()), fixture(t, "zfs-list.txt")) + f.on(line("zfs", SnapshotsArgv()), fixture(t, "zfs-list-snapshots.txt")) + f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list.txt")) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status.txt")) + return f +} + +func ran(f *fake, prefix string) []string { + var out []string + for _, c := range f.calls { + if strings.HasPrefix(c, prefix) { + out = append(out, c) + } + } + return out +} diff --git a/modules/zfs/cmd/zfs-tools/io.go b/modules/zfs/cmd/zfs-tools/io.go new file mode 100644 index 00000000..0073b512 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/io.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" +) + +// IOArgv samples every pool's I/O for some seconds: -y leaves out the report since boot, so the one report +// is the window alone. +func IOArgv(seconds int) []string { + return []string{"iostat", "-y", "-Hpv", strconv.Itoa(seconds), "1"} +} + +// IORow is one line of the sample: a pool, a vdev, a device or a section. +type IORow struct { + Name string `json:"name"` + Pool string `json:"pool"` + // Kind is pool, vdev, device or section, from the pool's layout. + Kind string `json:"kind"` + Allocated *uint64 `json:"allocated_bytes,omitempty"` + Free *uint64 `json:"free_bytes,omitempty"` + ReadOps float64 `json:"read_ops_per_s"` + WriteOps float64 `json:"write_ops_per_s"` + ReadBytes float64 `json:"read_bytes_per_s"` + WriteBytes float64 `json:"write_bytes_per_s"` +} + +// ParseIO reads `zpool iostat -Hpv`: name, allocated, free, read and write operations, read and write +// bandwidth, tab-separated; "-" where a line has no such number. Lines of a pool follow the pool's own +// line; layout names each line's kind. +func ParseIO(out string, layout []PoolStatus) ([]IORow, error) { + var rows []IORow + pool := "" + for _, line := range lines(out) { + f := strings.Split(line, "\t") + if len(f) != 7 { + f = strings.Fields(line) + } + if len(f) != 7 { + return nil, fmt.Errorf("zpool iostat answered a line with %d columns, not 7: %q", len(f), line) + } + r := IORow{Name: strings.TrimSpace(f[0])} + if statusOf(layout, r.Name) != nil { + pool = r.Name + r.Kind = "pool" + } + r.Pool = pool + if r.Kind == "" { + r.Kind = "device" + if _, ok := sections[r.Name]; ok && allDash(f[1:]) { + r.Kind = "section" + } else if p := statusOf(layout, pool); p != nil { + walk(p.Root, func(d *Device, _ int) { + if d.Name == r.Name && d.Kind != "pool" { + r.Kind = d.Kind + } + }, 0) + } + } + r.Allocated, r.Free = optionalNumber(f[1]), optionalNumber(f[2]) + vals := []*float64{&r.ReadOps, &r.WriteOps, &r.ReadBytes, &r.WriteBytes} + for i, v := range vals { + s := f[3+i] + if s == "-" { + continue + } + x, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, fmt.Errorf("zpool iostat: %q of %s is not a number", s, r.Name) + } + *v = x + } + rows = append(rows, r) + } + return rows, nil +} + +func allDash(f []string) bool { + for _, s := range f { + if strings.TrimSpace(s) != "-" { + return false + } + } + return true +} + +func optionalNumber(s string) *uint64 { + if s == "-" { + return nil + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil + } + return &v +} + +// IOAnswer is what zfs_io answers. +type IOAnswer struct { + Applicable bool `json:"applicable"` + Seconds int `json:"window_s"` + Rows []IORow `json:"rows"` + Said []string `json:"said"` +} + +// IO samples every pool's operations and bandwidth for a few seconds. +func (m *Machine) IO(ctx context.Context, seconds int) (any, error) { + st, err := m.Status(ctx) + if err != nil { + return notHere(err) + } + sctx, cancel := context.WithTimeout(ctx, time.Duration(seconds)*time.Second+CommandTimeout) + defer cancel() + out, err := m.ask(sctx, "zpool", IOArgv(seconds)...) + if err != nil { + return notHere(err) + } + rows, err := ParseIO(out, st) + if err != nil { + return nil, err + } + a := IOAnswer{Applicable: true, Seconds: seconds, Rows: rows, Said: []string{}} + for _, r := range rows { + if r.Kind == "section" { + a.Said = append(a.Said, r.Pool+" "+r.Name+":") + continue + } + a.Said = append(a.Said, fmt.Sprintf("%s %s (%s): read %.0f ops/s %s/s, write %.0f ops/s %s/s", r.Pool, r.Name, r.Kind, + r.ReadOps, bytesSaid(uint64(r.ReadBytes)), r.WriteOps, bytesSaid(uint64(r.WriteBytes)))) + } + return a, nil +} + +// ArcStats is where the kernel module keeps the ARC's counters. +const ArcStats = "/proc/spl/kstat/zfs/arcstats" + +// ParseArc reads arcstats: two header lines, then `name type value`. +func ParseArc(text string) map[string]uint64 { + out := map[string]uint64{} + for i, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if i < 2 || len(f) != 3 { + continue + } + if v, err := strconv.ParseUint(f[2], 10, 64); err == nil { + out[f[0]] = v + } + } + return out +} + +// ArcAnswer is what zfs_arc answers. +type ArcAnswer struct { + Applicable bool `json:"applicable"` + Size uint64 `json:"size_bytes"` + Target uint64 `json:"target_bytes"` + Min uint64 `json:"min_bytes"` + Max uint64 `json:"max_bytes"` + Hits uint64 `json:"hits"` + Misses uint64 `json:"misses"` + HitRatio float64 `json:"hit_ratio_pct"` + // DemandHitRatio counts only what was asked for, not what ZFS read ahead. + DemandHitRatio *float64 `json:"demand_hit_ratio_pct,omitempty"` + L2 *L2Arc `json:"l2arc,omitempty"` + Said []string `json:"said"` +} + +// L2Arc is the cache device's share of the ARC, where the pool has one. +type L2Arc struct { + Size uint64 `json:"size_bytes"` + Allocated uint64 `json:"allocated_bytes"` + Hits uint64 `json:"hits"` + Misses uint64 `json:"misses"` + HitRatio float64 `json:"hit_ratio_pct"` +} + +// Arc is the ARC's size and hit ratio since the machine started. +func (m *Machine) Arc() (any, error) { + raw, err := m.ReadFile(ArcStats) + if err != nil { + if os.IsNotExist(err) { + return NotApplicable{Applicable: false, Said: "there is no " + ArcStats + ": ZFS's kernel module is not loaded on this machine, so this module does not apply here"}, nil + } + return nil, fmt.Errorf("%s could not be read: %v", ArcStats, err) + } + s := ParseArc(string(raw)) + if _, ok := s["size"]; !ok { + return nil, fmt.Errorf("%s has no size line", ArcStats) + } + a := ArcAnswer{Applicable: true, Size: s["size"], Target: s["c"], Min: s["c_min"], Max: s["c_max"], Hits: s["hits"], Misses: s["misses"]} + a.HitRatio = pct(a.Hits, a.Hits+a.Misses) + dh := s["demand_data_hits"] + s["demand_metadata_hits"] + dm := s["demand_data_misses"] + s["demand_metadata_misses"] + if dh+dm > 0 { + r := pct(dh, dh+dm) + a.DemandHitRatio = &r + } + a.Said = []string{fmt.Sprintf("ARC: %s of a %s target (min %s, max %s); %.1f %% of %d reads since the machine started were hits", + bytesSaid(a.Size), bytesSaid(a.Target), bytesSaid(a.Min), bytesSaid(a.Max), a.HitRatio, a.Hits+a.Misses)} + if s["l2_size"] > 0 || s["l2_hits"]+s["l2_misses"] > 0 { + l := &L2Arc{Size: s["l2_size"], Allocated: s["l2_asize"], Hits: s["l2_hits"], Misses: s["l2_misses"]} + l.HitRatio = pct(l.Hits, l.Hits+l.Misses) + a.L2 = l + a.Said = append(a.Said, fmt.Sprintf("L2ARC: %s cached (%s on the device), %.1f %% of %d reads were hits", + bytesSaid(l.Size), bytesSaid(l.Allocated), l.HitRatio, l.Hits+l.Misses)) + } else { + a.Said = append(a.Said, "no L2ARC (no cache device)") + } + return a, nil +} + +func pct(part, whole uint64) float64 { + if whole == 0 { + return 0 + } + return round1(float64(part) * 100 / float64(whole)) +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go new file mode 100644 index 00000000..95c115a0 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -0,0 +1,186 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// The machine as the module sees it. +// +// **It runs fixed commands, never a shell.** Every command is a program and an argument list built here; +// nothing an argument says is ever read by a shell. The tools run as the operator account (novox/hq ADR +// 0175 §4). What ZFS lets any account read — `zfs list`, `zfs get`, `zpool list`, `zpool status`, +// `zpool iostat` — runs as that account. What ZFS keeps for root — `zpool events`, `zpool history` and +// `zfs snapshot` — goes through `sudo -n`, as the packet filter's, the service manager's and nfs-server's +// tools do: without a prompt, so a refusal is said rather than waited on. + +// CommandTimeout bounds every command but a sample. +const CommandTimeout = 30 * time.Second + +// Ran is what one command did. +type Ran struct { + Out string + Err string + Code int +} + +// Runner runs one command; injected, so every tool is tested over recorded answers. +type Runner func(ctx context.Context, name string, args ...string) Ran + +// ExecRunner runs a command on the machine, bounded by the context (CommandTimeout unless the caller set +// one). +func ExecRunner(ctx context.Context, name string, args ...string) Ran { + if _, has := ctx.Deadline(); !has { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, CommandTimeout) + defer cancel() + } + cmd := exec.CommandContext(ctx, name, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + r := Ran{Out: stdout.String(), Err: strings.TrimSpace(stderr.String())} + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + r.Code = exit.ExitCode() + } else { + r.Code = -1 + if r.Err == "" { + r.Err = err.Error() + } + } + if ctx.Err() != nil { + r.Code = -1 + r.Err = fmt.Sprintf("did not answer within its time (%v)", ctx.Err()) + } + } + return r +} + +// Machine is what the module reads and the one thing it does. +type Machine struct { + Run Runner + // UID is this process's account: root runs a command as it is, any other account through sudo -n. + UID int + Now func() time.Time + // ReadFile reads a kernel file (the ARC's counters). + ReadFile func(path string) ([]byte, error) + // Emit publishes an event as this module (the record of an act); nil where nothing can be published. + Emit func(eventType string, body any) error +} + +// Here is the machine this process runs on. +func Here() *Machine { + return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile} +} + +// NotApplicable is the answer of a machine without ZFS: the module does not apply here. +type NotApplicable struct { + Applicable bool `json:"applicable"` + Said string `json:"said"` +} + +// absent says why ZFS cannot be asked on this machine, from a command's failure; "" when the failure is +// something else. +func absent(r Ran) string { + switch { + case r.Code == -1 && (strings.Contains(r.Err, "executable file not found") || strings.Contains(r.Err, "no such file or directory")): + return "ZFS is not installed on this machine (there is no zfs or zpool command), so this module does not apply here" + case strings.Contains(r.Err, "/dev/zfs") || strings.Contains(r.Err, "ZFS modules cannot be auto-loaded") || + strings.Contains(r.Err, "modprobe zfs"): + return "ZFS's kernel module is not loaded on this machine (" + firstLine(r.Err) + "), so there is no pool to read" + } + return "" +} + +// errAbsent is the error a tool answers when ZFS is not here. +type errAbsent struct{ why string } + +func (e errAbsent) Error() string { return e.why } + +// ask runs a read-only command as this account and answers its output, or why it could not. +func (m *Machine) ask(ctx context.Context, name string, args ...string) (string, error) { + r := m.Run(ctx, name, args...) + if r.Code != 0 { + if why := absent(r); why != "" { + return "", errAbsent{why} + } + return "", fmt.Errorf("%s %s: exit %d: %s", name, strings.Join(args, " "), r.Code, firstLine(orOut(r))) + } + return r.Out, nil +} + +// Escalated is a command as it is run: as given when this process is root, else through sudo without a +// prompt. +func (m *Machine) Escalated(name string, args ...string) (string, []string) { + if m.UID == 0 { + return name, args + } + return "sudo", append([]string{"-n", name}, args...) +} + +// AsRoot runs what only root may do, and names a refusal by how it failed. +func (m *Machine) AsRoot(ctx context.Context, name string, args ...string) (Ran, error) { + program, argv := m.Escalated(name, args...) + r := m.Run(ctx, program, argv...) + if r.Code != 0 { + why := orOut(r) + if why2 := absent(r); why2 != "" && program != "sudo" { + return r, errAbsent{why2} + } + if program == "sudo" && (strings.Contains(why, "a password is required") || strings.Contains(why, "not allowed") || + strings.Contains(why, "may not run sudo")) { + return r, fmt.Errorf("%s needs root, and sudo refused the operator account without a prompt: %s", name, firstLine(why)) + } + if program == "sudo" && strings.Contains(why, "command not found") { + return r, errAbsent{"ZFS is not installed on this machine (sudo found no " + name + " command), so this module does not apply here"} + } + return r, fmt.Errorf("%s %s: exit %d: %s", name, strings.Join(args, " "), r.Code, firstLine(why)) + } + return r, nil +} + +func orOut(r Ran) string { + if strings.TrimSpace(r.Err) != "" { + return r.Err + } + return strings.TrimSpace(r.Out) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +// bytesSaid says a size for a person: 1.2 TiB, 512 MiB, 900 B. +func bytesSaid(n uint64) string { + units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"} + f := float64(n) + i := 0 + for f >= 1024 && i < len(units)-1 { + f /= 1024 + i++ + } + if i == 0 { + return fmt.Sprintf("%d B", n) + } + return fmt.Sprintf("%.1f %s", f, units[i]) +} + +// round1 rounds to one decimal, for what a person reads. +func round1(f float64) float64 { + if f < 0 { + return -float64(int64(-f*10+0.5)) / 10 + } + return float64(int64(f*10+0.5)) / 10 +} diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go new file mode 100644 index 00000000..f1ba8ab7 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -0,0 +1,45 @@ +// The zfs module's Go bundle: the mesh's view of ZFS on a machine — its pools' health, layout, space, +// scrubs, I/O, cache and events, every dataset with its space and snapshots — and one act, taking a named +// snapshot of one dataset. +// +// One binary, three ways to run. With no argument it is the bundle the node's tool runner launches and +// speaks MCP to over stdio (novox/hq ADR 0188, ADR 0193), as the operator account. With `watch` and +// `events` it is one of the module's two long-running processes, run by the node-engine: the watch, which +// the health check zfs_health stands on (watch.go), and the event follower, as root, judged by its unit +// (follow.go). +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func main() { + if len(os.Args) > 1 { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) + defer stop() + switch os.Args[1] { + case "watch": + (&Watch{M: Here(), Out: os.Stdout}).Run(ctx) + case "events": + Follower(ctx, os.Stdout) + default: + fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+ + "watches the pools, with events it follows ZFS's events\n", os.Args[1]) + os.Exit(2) + } + return + } + m := Here() + m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) } + // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. + if err := stdio.Serve("", Tools(m)); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go new file mode 100644 index 00000000..4fae4d80 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +type manifest struct { + Tools []string `json:"tools"` + Claims []any `json:"claims"` + Emits []string `json:"emits"` + Replaces map[string][]string `json:"replaces"` + Resources []map[string]any `json:"resources"` +} + +func readManifest(t *testing.T) (manifest, string) { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m, string(raw) +} + +// The manifest lists exactly the tools the bundle serves; every one but zfs_snapshot says it only reads, +// and zfs_snapshot says it acts. Each says what it replaces (novox/hq ADR 0245), but the health check, +// which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121). +func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) { + m, raw := readManifest(t) + var served []string + for _, tl := range Tools(newFake(t).machine()) { + served = append(served, tl.Name) + want := "(r)" + if tl.Name == "zfs_snapshot" { + want = "(a)" + } + if !strings.HasSuffix(tl.Description, want) { + t.Errorf("%s does not end with %s", tl.Name, want) + } + if tl.Name != "zfs_health" && len(m.Replaces[tl.Name]) == 0 { + t.Errorf("%s says nothing of what it replaces", tl.Name) + } + for _, c := range m.Replaces[tl.Name] { + if !strings.Contains(tl.Description, c) { + t.Errorf("%s replaces %q in the manifest and not in its description", tl.Name, c) + } + } + } + sort.Strings(served) + listed := append([]string(nil), m.Tools...) + sort.Strings(listed) + if strings.Join(served, ",") != strings.Join(listed, ",") { + t.Fatalf("served %v, listed %v", served, listed) + } + if _, ok := m.Replaces["zfs_health"]; ok { + t.Errorf("the health check replaces no command") + } + if len(m.Claims) != 0 { + t.Errorf("the module claims no seat: %v", m.Claims) + } + if strings.Join(m.Emits, ",") != SnapshotEvent { + t.Errorf("emits %v", m.Emits) + } + if hit := regexp.MustCompile(`\b(g14|novox|ace|shanks|jochen)\b|/home/`).FindString(strings.ReplaceAll(raw, "novox/hq", "")); hit != "" { + t.Errorf("the manifest says %q", hit) + } +} + +// ADR 0240: the health is zfs_health on the watch, beside the event follower's unit, which the module does +// not judge itself; the module installs nothing, so it never puts ZFS on a machine. +func TestTheHealthIsTheWatchsToolCheckBesideTheFollowersUnit(t *testing.T) { + m, raw := readManifest(t) + byID := map[string]map[string]any{} + for _, r := range m.Resources { + byID[r["id"].(string)] = r + } + if len(byID) != 2 { + t.Fatalf("resources: %v", m.Resources) + } + for id, want := range map[string][2]string{"watch": {"tool", "watch"}, "events": {"unit", "events"}} { + r := byID[id] + h, _ := r["health"].(map[string]any) + run, _ := r["run"].([]any) + if r["type"] != "process" || h["kind"] != want[0] || len(run) != 2 || run[1] != want[1] { + t.Errorf("%s: %v", id, r) + } + } + if h := byID["watch"]["health"].(map[string]any); h["tool"] != "zfs_health" { + t.Errorf("the watch is judged by zfs_health: %v", h) + } + if strings.Contains(raw, `"package"`) { + t.Errorf("the module installs no package: ZFS is the machine's, and its kernel module is built against its kernel") + } +} + +// No destroy, rollback, set, send or any other change but a snapshot. Every command the bundle runs goes +// through ask, AsRoot or Escalated with an argument list from one of the functions below, each of which is +// held to its verb here; a call site that builds its own list fails this test. A new verb needs a decision +// first, and then a line here. +func TestTheBundleRunsOnlyReadingVerbsAndSnapshot(t *testing.T) { + allowed := map[string]string{"DatasetsArgv": "list", "SnapshotsArgv": "list", "PoolsArgv": "list", "StatusArgv": "status", + "IOArgv": "iostat", "EventsArgv": "events", "HistoryArgv": "history", "PropertiesArgv": "get", "SnapshotArgv": "snapshot", "FollowArgv": "events"} + verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), + "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), + "PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x", true), "FollowArgv": FollowArgv()} + for fn, argv := range verbs { + if argv[0] != allowed[fn] { + t.Errorf("%s runs %q, not %q", fn, argv[0], allowed[fn]) + } + } + call := regexp.MustCompile(`\.(ask|AsRoot|Escalated|Command|CommandContext)\(`) + site := regexp.MustCompile(`\.(?:ask|AsRoot|Escalated|CommandContext)\((?:\w+, )?"(zfs|zpool)", (\w+)\(`) + files, _ := filepath.Glob("*.go") + sites := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + b, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for i, l := range strings.Split(string(b), "\n") { + if !call.MatchString(l) || strings.HasPrefix(strings.TrimSpace(l), "func ") || strings.HasPrefix(strings.TrimSpace(l), "//") { + continue + } + if strings.Contains(l, "m.Escalated(name, args...)") || (file == "machine.go" && strings.Contains(l, "exec.CommandContext(ctx, name, args...)")) { + continue // AsRoot's and the runner's own bodies + } + m := site.FindStringSubmatch(l) + if m == nil { + t.Errorf("%s:%d runs a command not built by an argument function: %s", file, i+1, strings.TrimSpace(l)) + continue + } + if _, ok := allowed[m[2]]; !ok { + t.Errorf("%s:%d runs %s's list, which is not held here", file, i+1, m[2]) + } + sites++ + } + } + if sites < 10 { + t.Errorf("found %d call sites; the test no longer sees them", sites) + } +} diff --git a/modules/zfs/cmd/zfs-tools/parse_test.go b/modules/zfs/cmd/zfs-tools/parse_test.go new file mode 100644 index 00000000..08f66c33 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/parse_test.go @@ -0,0 +1,403 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestDatasetsAreReadFromZfsListWithEverySizeInBytes(t *testing.T) { + ds, err := ParseDatasets(fixture(t, "zfs-list.txt")) + if err != nil { + t.Fatal(err) + } + if len(ds) != 4 { + t.Fatalf("datasets: %d", len(ds)) + } + media := find(ds, "tank/media") + if media == nil || media.Mountpoint != "/storage/media" || media.Used != 27487790694400 || media.Available != 11203942195200 || + media.Referenced != 26938034880512 || media.UsedBySnapshots != 549755813888 || media.ShareNFS != "rw=@192.0.2.0/24" || + media.CompressRatio != 1.00 || media.Pool != "tank" || media.Compression != "lz4" { + t.Errorf("tank/media: %+v", media) + } + if media.Quota != nil || media.Reservation != nil { + t.Errorf("a quota of 0 under -p is none: %+v", media) + } + dl := find(ds, "tank/downloads") + if dl.Quota == nil || *dl.Quota != 2199023255552 { + t.Errorf("tank/downloads has a 2 TiB quota: %+v", dl.Quota) + } + vm := find(ds, "tank/vm") + if vm.Type != "volume" || vm.Mountpoint != "" || vm.ShareNFS != "" || vm.CompressRatio != 1.85 || + vm.RefReservation == nil || *vm.RefReservation != 4398046511 || vm.UsedByRefreservation != 2250562863 || vm.Quota != nil { + t.Errorf("a volume has no mountpoint or sharenfs, and its refreservation is read: %+v", vm) + } +} + +func TestALineWithoutEveryColumnIsRefusedNotSkipped(t *testing.T) { + if _, err := ParseDatasets("tank\tfilesystem\t/storage\n"); err == nil || !strings.Contains(err.Error(), "3 columns") { + t.Errorf("a short line: %v", err) + } + if _, err := ParseDatasets(strings.Replace(fixture(t, "zfs-list.txt"), "27487790694400", "25.0T", 1)); err == nil { + t.Errorf("a size that is not whole bytes is refused (the command lost its -p)") + } + if _, err := ParsePools("tank\t1\t2\n"); err == nil { + t.Errorf("a short pool line is refused") + } +} + +func TestSnapshotsCarryTheirCreationTimeAndTheirOwnSpace(t *testing.T) { + ss, err := ParseSnapshots(fixture(t, "zfs-list-snapshots.txt")) + if err != nil { + t.Fatal(err) + } + if len(ss) != 4 { + t.Fatalf("snapshots: %d", len(ss)) + } + s := ss[0] + if s.Dataset != "tank/media" || s.Snap != "2026-08-01" || !s.Created.Equal(time.Unix(1785542400, 0)) || + s.Used != 274877906944 || s.Referenced != 26388279066624 { + t.Errorf("first snapshot: %+v", s) + } +} + +func TestPoolsAreReadFromZpoolList(t *testing.T) { + ps, err := ParsePools(fixture(t, "zpool-list.txt")) + if err != nil { + t.Fatal(err) + } + if len(ps) != 2 || ps[0].Name != "tank" || ps[0].Capacity != 60 || ps[0].Fragmentation == nil || *ps[0].Fragmentation != 3 || + ps[0].Health != "ONLINE" || ps[0].Size != 47982397226598 || ps[0].Free != 19184805568512 { + t.Fatalf("pools: %+v", ps) + } + if ps[1].Fragmentation != nil { + t.Errorf("a pool whose fragmentation is '-' has none said: %v", *ps[1].Fragmentation) + } +} + +func TestStatusGivesEachPoolsLayoutWithSectionsAndTheLastScrub(t *testing.T) { + st, err := ParseStatus(fixture(t, "zpool-status.txt")) + if err != nil { + t.Fatal(err) + } + if len(st) != 2 || st[0].Name != "rpool" || st[1].Name != "tank" { + t.Fatalf("pools: %+v", st) + } + r := st[0] + if !strings.HasPrefix(r.Status, "Some supported and requested features") || !strings.HasSuffix(r.Status, "some features are unavailable.") { + t.Errorf("a status of two lines is one sentence: %q", r.Status) + } + tank := st[1] + if tank.State != "ONLINE" || tank.Errors != "No known data errors" || tank.Root == nil || tank.Root.Kind != "pool" { + t.Fatalf("tank: %+v", tank) + } + kids := tank.Root.Children + if len(kids) != 4 { + t.Fatalf("tank's top level is raidz2-0, logs, cache and spares: %+v", kids) + } + if kids[0].Name != "raidz2-0" || kids[0].Kind != "vdev" || kids[0].Role != "data" || len(kids[0].Children) != 4 || + kids[0].Children[0].Name != "ata-WDC_WD120EFBX-68B0EN0_D7JA0001" || kids[0].Children[0].Kind != "device" { + t.Errorf("raidz2-0: %+v", kids[0]) + } + if kids[1].Kind != "section" || kids[1].Role != "log" || len(kids[1].Children) != 1 || kids[1].Children[0].Name != "mirror-1" || + len(kids[1].Children[0].Children) != 2 || kids[1].Children[0].Children[1].Role != "log" { + t.Errorf("logs: %+v", kids[1]) + } + if kids[2].Role != "cache" || kids[2].Children[0].Kind != "device" || kids[2].Children[0].State != "ONLINE" { + t.Errorf("cache: %+v", kids[2]) + } + if kids[3].Role != "spare" || kids[3].Children[0].State != "AVAIL" || kids[3].Children[0].Note != "" { + t.Errorf("spares: %+v", kids[3].Children[0]) + } + s := tank.Scan + end := time.Date(2026, 9, 14, 2, 34, 6, 0, time.Local) + if s.Kind != "scrub" || s.State != "finished" || s.Took != "1 days 02:10:05" || s.Repaired != "0" || deref(s.Errors) != 0 || + s.Ended == nil || !s.Ended.Equal(end) || s.Started == nil || !s.Started.Equal(end.Add(-26*time.Hour-10*time.Minute-5*time.Second)) { + t.Errorf("tank's scrub: %+v", s) + } +} + +func TestADegradedPoolSaysItsFaultedDeviceItsErrorsAndTheResilver(t *testing.T) { + st, err := ParseStatus(fixture(t, "zpool-status-degraded.txt")) + if err != nil { + t.Fatal(err) + } + p := st[0] + if p.State != "DEGRADED" || p.Action != "Wait for the resilver to complete." || p.Errors != "2 data errors, use '-v' for a list" { + t.Fatalf("pool: %+v", p) + } + spare := p.Root.Children[0].Children[1] + if spare.Name != "spare-1" || spare.Kind != "vdev" || len(spare.Children) != 2 { + t.Fatalf("spare-1 holds the failed disk and its spare: %+v", spare) + } + bad := spare.Children[0] + if bad.State != "FAULTED" || bad.Read != 12 || bad.Write != 107 || bad.Checksum != 3 || bad.Note != "too many errors" { + t.Errorf("the faulted disk: %+v", bad) + } + if spare.Children[1].Note != "(resilvering)" { + t.Errorf("the spare: %+v", spare.Children[1]) + } + if in := p.Root.Children[1].Children[0]; in.State != "INUSE" || in.Note != "currently in use" { + t.Errorf("the spare in use: %+v", in) + } + s := p.Scan + if s.Kind != "resilver" || s.State != "running" || s.Done == nil || *s.Done != 22.72 || s.ToGo != "08:25:13" || + s.Started == nil || !s.Started.Equal(time.Date(2026, 10, 8, 21, 2, 17, 0, time.Local)) { + t.Errorf("the resilver: %+v", s) + } +} + +func TestARunningScrubSaysItsProgress(t *testing.T) { + st, err := ParseStatus(fixture(t, "zpool-status-scrubbing.txt")) + if err != nil { + t.Fatal(err) + } + s := st[0].Scan + if s.Kind != "scrub" || s.State != "running" || *s.Done != 29.17 || s.ToGo != "06:49:34" || s.Repaired != "0" { + t.Errorf("scrub: %+v", s) + } + for _, said := range []string{"none requested", "scrub canceled on Sun Oct 4 01:00:00 2026", "scrub paused since Sun Oct 4 01:00:00 2026"} { + if got := parseScan(said); got.State == "unknown" { + t.Errorf("%q not read: %+v", said, got) + } + } + if got := parseScan("resilvered 1088516226 in 03:22:11 with 0 errors on Fri Oct 9 06:00:00 2026"); got.Kind != "resilver" || got.State != "finished" || got.Repaired != "1088516226" { + t.Errorf("a finished resilver: %+v", got) + } +} + +func TestTheHistoryGivesTheLastFinishedScrubAndResilverNotACanceledOne(t *testing.T) { + scrub, resilver := ParseHistory(fixture(t, "zpool-history.txt")) + if scrub == nil || !scrub.Ended.Equal(time.Date(2026, 7, 13, 2, 30, 0, 0, time.Local)) || deref(scrub.Errors) != 0 { + t.Errorf("the last finished scrub is July's; September's was canceled: %+v", scrub) + } + if resilver == nil || !resilver.Ended.Equal(time.Date(2026, 8, 20, 19, 40, 2, 0, time.Local)) { + t.Errorf("the last finished resilver: %+v", resilver) + } +} + +func TestIOSampleNamesEachLinesPoolAndKindFromTheLayout(t *testing.T) { + st, _ := ParseStatus(fixture(t, "zpool-status.txt")) + rows, err := ParseIO(fixture(t, "zpool-iostat.txt"), st) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"tank": "pool", "raidz2-0": "vdev", "ata-WDC_WD120EFBX-68B0EN0_D7JA0001": "device", "logs": "section", + "mirror-1": "vdev", "cache": "section", "rpool": "pool", "nvme-Samsung_SSD_970_EVO_Plus_500GB_S4EVNX0R200001-part2": "device"} + for _, r := range rows { + if k, ok := want[r.Name]; ok && r.Kind != k { + t.Errorf("%s is a %s, said %s", r.Name, k, r.Kind) + } + } + last := rows[len(rows)-1] + if last.Pool != "rpool" || last.WriteOps != 25 || last.WriteBytes != 1310720 { + t.Errorf("the last line is rpool's disk: %+v", last) + } + if rows[2].Allocated != nil || rows[0].ReadBytes != 221249536 { + t.Errorf("a disk of a raidz has no allocation of its own: %+v / %+v", rows[2], rows[0]) + } +} + +func TestTheArcIsReadFromArcstats(t *testing.T) { + f := newFake(t) + f.files[ArcStats] = fixture(t, "arcstats.txt") + got, err := f.machine().Arc() + if err != nil { + t.Fatal(err) + } + a := got.(ArcAnswer) + if a.Size != 16798000128 || a.Target != 16823304192 || a.Max != 33646608384 || a.HitRatio != 90.5 || + a.DemandHitRatio == nil || *a.DemandHitRatio != 93.0 || a.L2 == nil || a.L2.HitRatio != 25.0 || a.L2.Size != 412316860416 { + t.Errorf("arc: %+v %+v", a, a.L2) + } + f.files[ArcStats] = fixture(t, "arcstats-nol2.txt") + got, _ = f.machine().Arc() + if a := got.(ArcAnswer); a.L2 != nil || a.HitRatio != 90 || !strings.Contains(strings.Join(a.Said, " "), "no L2ARC") { + t.Errorf("without a cache device: %+v", a) + } +} + +func TestEventsAreErrorsAndChangesOfStateNewestFirstAsRoot(t *testing.T) { + f := newFake(t) + f.on("sudo -n zpool events -v", fixture(t, "zpool-events.txt")) + got, err := f.machine().Events(context.Background(), false, 50) + if err != nil { + t.Fatal(err) + } + a := got.(EventsAnswer) + if a.Kept != 5 || a.Shown != 3 { + t.Fatalf("kept %d, shown %d: %+v", a.Kept, a.Shown, a.Events) + } + if a.Events[0].Class != "sysevent.fs.zfs.resilver_start" || a.Events[1].Class != "resource.fs.zfs.statechange" || + a.Events[2].Class != "ereport.fs.zfs.io" { + t.Errorf("newest first, history and config_sync left out: %+v", a.Events) + } + sc := a.Events[1].Details + if sc["vdev_state"] != "FAULTED" || sc["vdev_laststate"] != "ONLINE" || sc["pool"] != "tank" || + sc["vdev_path"] != "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_D7JA0002-part1" { + t.Errorf("the state change: %+v", sc) + } + if _, kept := a.Events[2].Details["ena"]; kept { + t.Errorf("internal identifiers are left out: %+v", a.Events[2].Details) + } + got, _ = f.machine().Events(context.Background(), true, 2) + if a := got.(EventsAnswer); a.Shown != 2 || a.Events[1].Class != "sysevent.fs.zfs.config_sync" { + t.Errorf("all, limited to 2: %+v", a.Events) + } +} + +func TestPropertiesWithholdWhereTheKeyIsAndValidateTheDataset(t *testing.T) { + f := withZFS(t) + f.on(line("zfs", PropertiesArgv("tank/media")), fixture(t, "zfs-get.txt")) + got, err := f.machine().PropertiesOf(context.Background(), "tank/media") + if err != nil { + t.Fatal(err) + } + a := got.(PropertiesAnswer) + by := map[string]Property{} + for _, p := range a.Properties { + by[p.Name] = p + } + if strings.Contains(by["keylocation"].Value, "s3cret") || by["encryption"].Value != "aes-256-gcm" || by["keystatus"].Value != "available" || + by["tank:purpose"].Value != "the library" || by["recordsize"].Source != "local" { + t.Errorf("properties: %+v", by) + } + said := strings.Join(a.Said, "\n") + if !strings.Contains(said, "compression = lz4 (inherited from tank)") || strings.Contains(said, "s3cret") { + t.Errorf("said: %s", said) + } + if _, err := f.machine().PropertiesOf(context.Background(), "tank/nope"); err == nil { + t.Errorf("an unknown dataset is refused") + } + if n := len(ran(f, "zfs get")); n != 1 { + t.Errorf("zfs get runs only for a dataset that exists: %d", n) + } + if argv := PropertiesArgv("tank/media"); argv[len(argv)-2] != "--" { + t.Errorf("the dataset follows --: %v", argv) + } +} + +func TestDatasetsCountEachOnesSnapshots(t *testing.T) { + got, err := withZFS(t).machine().DatasetsNow(context.Background()) + if err != nil { + t.Fatal(err) + } + a := got.(DatasetsAnswer) + if a.Count != 4 || a.Datasets[1].Name != "tank/media" || a.Datasets[1].Snapshots != 3 || a.Datasets[2].Snapshots != 1 || + a.Datasets[0].Snapshots != 0 || a.Datasets[1].UsedBySnapshots != 549755813888 { + t.Errorf("datasets: %+v", a.Datasets) + } + if !strings.Contains(a.Said[1], "3 snapshots holding 512.0 GiB") || !strings.Contains(a.Said[1], "sharenfs rw=@192.0.2.0/24") { + t.Errorf("said: %s", a.Said[1]) + } +} + +func TestSpaceIsBiggestFirstWithTheBreakdownAndTheSavings(t *testing.T) { + got, err := withZFS(t).machine().Space(context.Background()) + if err != nil { + t.Fatal(err) + } + a := got.(SpaceAnswer) + if len(a.Pools) != 2 || a.Pools[0].Capacity != 60 { + t.Errorf("pools: %+v", a.Pools) + } + if a.Datasets[0].Name != "tank" || a.Datasets[1].Name != "tank/media" || a.Datasets[3].Name != "tank/vm" { + t.Errorf("biggest first: %v, %v, %v", a.Datasets[0].Name, a.Datasets[1].Name, a.Datasets[3].Name) + } + media := a.Datasets[1] + if media.Saved != 27500000000000-27487790694400 || media.UsedBySnapshots != 549755813888 { + t.Errorf("media: %+v", media) + } + if dl := a.Datasets[2]; dl.Quota == nil || !strings.Contains(dl.Said, "quota 2.0 TiB") { + t.Errorf("downloads: %+v", dl) + } +} + +func TestSnapshotsOfOneDatasetAreItsOwnOnly(t *testing.T) { + f := withZFS(t) + got, err := f.machine().SnapshotsOf(context.Background(), "tank/media") + if err != nil { + t.Fatal(err) + } + a := got.(SnapshotsAnswer) + if a.Count != 3 || a.Snapshots[2].Name != "tank/media@before-rename" || a.Unique != 274877906944+137438953472+1048576 { + t.Errorf("snapshots: %+v", a) + } + if _, err := f.machine().SnapshotsOf(context.Background(), "tank/med"); err == nil { + t.Errorf("an unknown dataset is refused") + } + got, _ = f.machine().SnapshotsOf(context.Background(), "") + if got.(SnapshotsAnswer).Count != 4 { + t.Errorf("every snapshot: %+v", got) + } +} + +func TestLayoutAndPoolsSayDeviceErrorsByName(t *testing.T) { + f := newFake(t) + f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list-full.txt")) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + got, err := f.machine().Layout(context.Background()) + if err != nil { + t.Fatal(err) + } + said := strings.Join(got.(LayoutAnswer).Said, "\n") + if !strings.Contains(said, "ata-WDC_WD120EFBX-68B0EN0_D7JA0002 FAULTED — ERRORS read 12, write 107, checksum 3 too many errors") { + t.Errorf("layout: %s", said) + } + got, err = f.machine().PoolsNow(context.Background()) + if err != nil { + t.Fatal(err) + } + p := got.(PoolsAnswer).Pools[0] + if p.State != "DEGRADED" || p.DeviceErrs != 122 || p.Capacity != 91 || !strings.HasPrefix(p.Scan, "resilver in progress") { + t.Errorf("pool: %+v", p) + } +} + +func TestIOAndScrubAnswerFromFakes(t *testing.T) { + f := withZFS(t) + f.on(line("zpool", IOArgv(3)), fixture(t, "zpool-iostat.txt")) + got, err := f.machine().IO(context.Background(), 3) + if err != nil { + t.Fatal(err) + } + if a := got.(IOAnswer); len(a.Rows) != 12 || a.Seconds != 3 { + t.Errorf("io: %+v", a) + } + if argv := IOArgv(3); strings.Join(argv, " ") != "iostat -y -Hpv 3 1" { + t.Errorf("argv: %v", argv) + } + got, err = f.machine().Scrubs(context.Background(), false) + if err != nil { + t.Fatal(err) + } + a := got.(ScrubAnswer) + rp, tank := a.Pools[0], a.Pools[1] + if tank.DaysSince == nil || *tank.DaysSince != 25.4 || tank.Warning != "" { + t.Errorf("tank was scrubbed 25 days ago: %+v", tank) + } + if rp.Warning == "" || !strings.Contains(rp.Said, "WARNING") { + t.Errorf("rpool was scrubbed in July, more than 35 days ago: %+v", rp) + } + if len(ran(f, "sudo")) != 0 { + t.Errorf("without history nothing runs as root: %v", f.calls) + } +} + +func TestScrubWithHistoryReadsItAsRootAndFindsTheScrubStatusNoLongerShows(t *testing.T) { + f := newFake(t) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + f.on("sudo -n zpool history -i tank", fixture(t, "zpool-history.txt")) + got, err := f.machine().Scrubs(context.Background(), true) + if err != nil { + t.Fatal(err) + } + r := got.(ScrubAnswer).Pools[0] + if r.LastScrub == nil || !r.LastScrub.Ended.Equal(time.Date(2026, 7, 13, 2, 30, 0, 0, time.Local)) || r.Warning == "" { + t.Errorf("the last scrub, from the history, is 88 days old: %+v", r) + } + if r.LastResilver == nil || r.Last.State != "running" { + t.Errorf("the resilver: %+v", r) + } +} diff --git a/modules/zfs/cmd/zfs-tools/pools.go b/modules/zfs/cmd/zfs-tools/pools.go new file mode 100644 index 00000000..c9ceac6d --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/pools.go @@ -0,0 +1,566 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// PoolColumns are the properties `zpool list` is asked for, in order. +var PoolColumns = []string{"name", "size", "allocated", "free", "capacity", "fragmentation", "health", "dedupratio"} + +// PoolsArgv lists every imported pool. +func PoolsArgv() []string { + return []string{"list", "-Hp", "-o", strings.Join(PoolColumns, ",")} +} + +// StatusArgv is every imported pool's status, numbers exact. +func StatusArgv() []string { return []string{"status", "-p"} } + +// Pool is one pool as `zpool list` answers it. +type Pool struct { + Name string `json:"name"` + Size uint64 `json:"size"` + Allocated uint64 `json:"allocated"` + Free uint64 `json:"free"` + // Capacity is the share of the pool allocated, in %. + Capacity int `json:"capacity_pct"` + // Fragmentation is the free space's fragmentation in %, nil when the pool cannot say. + Fragmentation *int `json:"fragmentation_pct,omitempty"` + Health string `json:"health"` + DedupRatio float64 `json:"dedupratio"` +} + +// ParsePools reads `zpool list -Hp -o `. +func ParsePools(out string) ([]Pool, error) { + var ps []Pool + for _, line := range lines(out) { + f := strings.Split(line, "\t") + if len(f) != len(PoolColumns) { + return nil, fmt.Errorf("zpool list answered a line with %d columns, not %d: %q", len(f), len(PoolColumns), line) + } + p := Pool{Name: f[0], Health: f[6]} + var err error + if p.Size, err = number(f[1]); err != nil { + return nil, fmt.Errorf("zpool list: size of %s: %v", p.Name, err) + } + if p.Allocated, err = number(f[2]); err != nil { + return nil, fmt.Errorf("zpool list: allocated of %s: %v", p.Name, err) + } + if p.Free, err = number(f[3]); err != nil { + return nil, fmt.Errorf("zpool list: free of %s: %v", p.Name, err) + } + if p.Capacity, err = strconv.Atoi(strings.TrimSuffix(f[4], "%")); err != nil { + return nil, fmt.Errorf("zpool list: capacity of %s is %q", p.Name, f[4]) + } + if fr, e := strconv.Atoi(strings.TrimSuffix(f[5], "%")); e == nil { + p.Fragmentation = &fr + } + if p.DedupRatio, err = ratio(f[7]); err != nil { + return nil, fmt.Errorf("zpool list: dedupratio of %s is %q", p.Name, f[7]) + } + ps = append(ps, p) + } + return ps, nil +} + +// PoolRow is one pool's space as the tools answer it. +type PoolRow struct { + Name string `json:"name"` + Health string `json:"health"` + Size uint64 `json:"size_bytes"` + Allocated uint64 `json:"allocated_bytes"` + Free uint64 `json:"free_bytes"` + Capacity int `json:"capacity_pct"` + Fragmentation *int `json:"fragmentation_pct,omitempty"` + Said string `json:"said"` +} + +// Row says a pool's space in a line. +func (p Pool) Row() PoolRow { + r := PoolRow{Name: p.Name, Health: p.Health, Size: p.Size, Allocated: p.Allocated, Free: p.Free, Capacity: p.Capacity, + Fragmentation: p.Fragmentation} + frag := "fragmentation not known" + if p.Fragmentation != nil { + frag = fmt.Sprintf("%d %% fragmented", *p.Fragmentation) + } + r.Said = fmt.Sprintf("pool %s (%s): %s of %s allocated (%d %%), %s free, %s", p.Name, p.Health, bytesSaid(p.Allocated), + bytesSaid(p.Size), p.Capacity, bytesSaid(p.Free), frag) + return r +} + +// Pools are every imported pool. +func (m *Machine) Pools(ctx context.Context) ([]Pool, error) { + out, err := m.ask(ctx, "zpool", PoolsArgv()...) + if err != nil { + return nil, err + } + return ParsePools(out) +} + +// Device is one line of a pool's configuration: the pool itself, a vdev (mirror, raidz), a disk, or a +// section (logs, cache, spares, special, dedup), with its children. +type Device struct { + Name string `json:"name"` + // Kind is pool, vdev (a group of disks), device (a disk or a partition), or section. + Kind string `json:"kind"` + // Role is data, log, cache, spare, special or dedup: the section a device is in. + Role string `json:"role,omitempty"` + State string `json:"state,omitempty"` + Read uint64 `json:"read_errors"` + Write uint64 `json:"write_errors"` + Checksum uint64 `json:"checksum_errors"` + Note string `json:"note,omitempty"` + Children []*Device `json:"children,omitempty"` +} + +// Errors is the error count status gives this line, read, write and checksum together: a vdev's line +// counts what was found at the vdev, not the sum of its disks. +func (d *Device) Errors() uint64 { return d.Read + d.Write + d.Checksum } + +// Scan is a pool's last scan, as status says it: ZFS keeps only the latest, a scrub's or a resilver's. +type Scan struct { + // Kind is scrub or resilver; "" when none was ever asked. + Kind string `json:"kind,omitempty"` + // State is finished, running, paused, canceled or none. + State string `json:"state"` + Started *time.Time `json:"started,omitempty"` + Ended *time.Time `json:"ended,omitempty"` + Took string `json:"took,omitempty"` + Repaired string `json:"repaired,omitempty"` + Errors *int `json:"errors,omitempty"` + Done *float64 `json:"done_pct,omitempty"` + ToGo string `json:"to_go,omitempty"` + Said string `json:"said"` +} + +// PoolStatus is one pool as `zpool status` says it. +type PoolStatus struct { + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status,omitempty"` + Action string `json:"action,omitempty"` + See string `json:"see,omitempty"` + Scan Scan `json:"scan"` + // Other are the lines status gives for a removal, a checkpoint or an expansion. + Other map[string]string `json:"other,omitempty"` + Root *Device `json:"layout,omitempty"` + Errors string `json:"errors"` +} + +var sections = map[string]string{"logs": "log", "cache": "cache", "spares": "spare", "special": "special", "dedup": "dedup"} + +// ParseStatus reads `zpool status -p` for every pool it shows. +func ParseStatus(out string) ([]PoolStatus, error) { + var pools []PoolStatus + var cur *PoolStatus + key := "" + inConfig := false + var stack []*Device + section := "" + flush := func() { + if cur != nil { + cur.Scan = parseScan(cur.Scan.Said) + pools = append(pools, *cur) + } + } + for _, raw := range strings.Split(out, "\n") { + line := strings.TrimRight(raw, "\r ") + trimmed := strings.TrimSpace(line) + if k, v, ok := statusKey(line); ok { + inConfig = false + key = k + switch k { + case "pool": + flush() + cur = &PoolStatus{Name: v} + stack, section = nil, "" + case "config": + inConfig = true + } + if cur == nil { + continue + } + switch k { + case "state": + cur.State = v + case "status": + cur.Status = v + case "action": + cur.Action = v + case "see": + cur.See = v + case "scan": + cur.Scan.Said = v + case "errors": + cur.Errors = v + case "pool", "config": + default: + if cur.Other == nil { + cur.Other = map[string]string{} + } + cur.Other[k] = v + } + continue + } + if cur == nil || trimmed == "" { + continue + } + if !inConfig { + // A continuation of the key before it, indented by a tab. + add := func(s *string) { *s = strings.TrimSpace(*s + " " + trimmed) } + switch key { + case "status": + add(&cur.Status) + case "action": + add(&cur.Action) + case "scan": + cur.Scan.Said += "\n" + trimmed + case "errors": + add(&cur.Errors) + default: + if cur.Other != nil { + if v, ok := cur.Other[key]; ok { + cur.Other[key] = strings.TrimSpace(v + " " + trimmed) + } + } + } + continue + } + // The configuration: a tab, two spaces per level, then the columns. + body := strings.TrimPrefix(line, "\t") + depth := (len(body) - len(strings.TrimLeft(body, " "))) / 2 + f := strings.Fields(body) + if f[0] == "NAME" && len(f) > 1 && f[1] == "STATE" { + continue + } + d := &Device{Name: f[0]} + if depth == 0 { + if role, ok := sections[f[0]]; ok && len(f) == 1 { + d.Kind, d.Role, section = "section", role, role + if cur.Root != nil { + cur.Root.Children = append(cur.Root.Children, d) + } + // A section stands at the pool's level and holds what is indented under it. + stack = []*Device{d} + continue + } + } + if len(f) > 1 { + d.State = f[1] + } + rest := f[min(len(f), 2):] + if len(rest) >= 3 && isCount(rest[0]) && isCount(rest[1]) && isCount(rest[2]) { + d.Read, d.Write, d.Checksum = count(rest[0]), count(rest[1]), count(rest[2]) + rest = rest[3:] + } + d.Note = strings.Join(rest, " ") + if depth == 0 { + d.Kind, d.Role = "pool", "" + cur.Root = d + stack, section = []*Device{d}, "" + continue + } + role := section + if role == "" { + role = "data" + } + d.Role = role + if depth < len(stack) { + stack = stack[:depth] + } + if len(stack) == 0 { + return nil, fmt.Errorf("zpool status: the device %s of pool %s has no parent", d.Name, cur.Name) + } + parent := stack[len(stack)-1] + parent.Children = append(parent.Children, d) + stack = append(stack, d) + } + flush() + for i := range pools { + kinds(pools[i].Root) + } + return pools, nil +} + +// kinds names each device by its place: a device with children is a vdev, one without a device. +func kinds(d *Device) { + if d == nil { + return + } + for _, c := range d.Children { + if c.Kind == "" { + if len(c.Children) > 0 { + c.Kind = "vdev" + } else { + c.Kind = "device" + } + } + kinds(c) + } +} + +var keyLine = regexp.MustCompile(`^\s{0,8}([a-z]+):(?:\s(.*))?$`) + +// statusKey is a `key: value` line of status: pool, state, status, action, see, scan, config, errors, … +func statusKey(line string) (string, string, bool) { + if strings.HasPrefix(line, "\t") { + return "", "", false + } + m := keyLine.FindStringSubmatch(line) + if m == nil { + return "", "", false + } + return m[1], strings.TrimSpace(m[2]), true +} + +func isCount(s string) bool { + if s == "" { + return false + } + _, err := strconv.ParseFloat(strings.TrimRight(s, "KMGTP"), 64) + return err == nil +} + +// count is an error count: exact under -p, else 1.2K and the like. +func count(s string) uint64 { + mult := 1.0 + switch s[len(s)-1] { + case 'K': + mult = 1e3 + case 'M': + mult = 1e6 + case 'G': + mult = 1e9 + case 'T': + mult = 1e12 + case 'P': + mult = 1e15 + } + f, _ := strconv.ParseFloat(strings.TrimRight(s, "KMGTP"), 64) + return uint64(f * mult) +} + +var ( + scanFinished = regexp.MustCompile(`^(scrub repaired|resilvered) (\S+) in (.+?) with (\d+) errors on (.+)$`) + scanRunning = regexp.MustCompile(`^(scrub|resilver) in progress since (.+)$`) + scanPaused = regexp.MustCompile(`^(scrub|resilver) paused since (.+)$`) + scanCanceled = regexp.MustCompile(`^(scrub|resilver) canceled on (.+)$`) + scanDone = regexp.MustCompile(`([\d.]+)% done`) + scanToGo = regexp.MustCompile(`(\d+ days \d+:\d+:\d+|\d+:\d+:\d+) to go`) + scanRepaired = regexp.MustCompile(`(\S+) repaired`) +) + +// parseScan reads status's scan line and its continuation. +func parseScan(said string) Scan { + s := Scan{State: "none", Said: said} + first, more, _ := strings.Cut(said, "\n") + first = strings.TrimSpace(first) + switch { + case first == "" || first == "none requested": + return s + case scanFinished.MatchString(first): + m := scanFinished.FindStringSubmatch(first) + s.Kind, s.State, s.Repaired, s.Took = "scrub", "finished", m[2], m[3] + if m[1] == "resilvered" { + s.Kind = "resilver" + } + n, _ := strconv.Atoi(m[4]) + s.Errors = &n + s.Ended = when(m[5]) + if s.Ended != nil { + if d, ok := took(m[3]); ok { + st := s.Ended.Add(-d) + s.Started = &st + } + } + case scanRunning.MatchString(first): + m := scanRunning.FindStringSubmatch(first) + s.Kind, s.State, s.Started = m[1], "running", when(m[2]) + case scanPaused.MatchString(first): + m := scanPaused.FindStringSubmatch(first) + s.Kind, s.State, s.Started = m[1], "paused", when(m[2]) + case scanCanceled.MatchString(first): + m := scanCanceled.FindStringSubmatch(first) + s.Kind, s.State, s.Ended = m[1], "canceled", when(m[2]) + default: + s.State = "unknown" + } + if more != "" { + if m := scanDone.FindStringSubmatch(more); m != nil { + v, _ := strconv.ParseFloat(m[1], 64) + s.Done = &v + } + if m := scanToGo.FindStringSubmatch(more); m != nil { + s.ToGo = m[1] + } + if m := scanRepaired.FindStringSubmatch(more); m != nil && s.Repaired == "" { + s.Repaired = m[1] + } + } + return s +} + +// when reads status's dates, as ctime writes them: "Sun Sep 13 05:36:34 2026", in local time. +func when(s string) *time.Time { + s = strings.TrimSpace(s) + for _, layout := range []string{time.ANSIC, "Mon Jan 2 15:04:05 2006"} { + if t, err := time.ParseInLocation(layout, s, time.Local); err == nil { + return &t + } + } + return nil +} + +// took reads a scan's duration: "05:12:33" or "1 days 05:12:33". +func took(s string) (time.Duration, bool) { + days := 0 + if d, rest, ok := strings.Cut(s, " days "); ok { + n, err := strconv.Atoi(d) + if err != nil { + return 0, false + } + days, s = n, rest + } + p := strings.Split(s, ":") + if len(p) != 3 { + return 0, false + } + var v [3]int + for i := range p { + n, err := strconv.Atoi(p[i]) + if err != nil { + return 0, false + } + v[i] = n + } + return time.Duration(days)*24*time.Hour + time.Duration(v[0])*time.Hour + time.Duration(v[1])*time.Minute + + time.Duration(v[2])*time.Second, true +} + +// Status is every pool's status. +func (m *Machine) Status(ctx context.Context) ([]PoolStatus, error) { + out, err := m.ask(ctx, "zpool", StatusArgv()...) + if err != nil { + return nil, err + } + return ParseStatus(out) +} + +// PoolsAnswer is what zfs_pools answers. +type PoolsAnswer struct { + Applicable bool `json:"applicable"` + Pools []PoolHealth `json:"pools"` + Said []string `json:"said"` +} + +// PoolHealth is one pool's health and space. +type PoolHealth struct { + PoolRow + State string `json:"state"` + Status string `json:"status,omitempty"` + Action string `json:"action,omitempty"` + Errors string `json:"errors"` + DeviceErrs uint64 `json:"device_errors"` + Scan string `json:"last_scan"` +} + +// PoolsNow is every pool's health in a line, with its space. +func (m *Machine) PoolsNow(ctx context.Context) (any, error) { + ps, err := m.Pools(ctx) + if err != nil { + return notHere(err) + } + st, err := m.Status(ctx) + if err != nil { + return nil, err + } + a := PoolsAnswer{Applicable: true, Pools: []PoolHealth{}, Said: []string{}} + for _, p := range ps { + h := PoolHealth{PoolRow: p.Row(), State: p.Health} + if s := statusOf(st, p.Name); s != nil { + h.State, h.Status, h.Action, h.Errors, h.Scan = s.State, s.Status, s.Action, s.Errors, firstLine(s.Scan.Said) + h.DeviceErrs = sumErrors(s.Root) + } + line := h.Said + fmt.Sprintf("; %d device errors; errors: %s", h.DeviceErrs, h.Errors) + if h.Status != "" { + line += "; status: " + h.Status + } + a.Pools = append(a.Pools, h) + a.Said = append(a.Said, line) + } + if len(ps) == 0 { + a.Said = append(a.Said, "ZFS is here, and no pool is imported.") + } + return a, nil +} + +// LayoutAnswer is what zfs_pool_layout answers. +type LayoutAnswer struct { + Applicable bool `json:"applicable"` + Pools []PoolStatus `json:"pools"` + Said []string `json:"said"` +} + +// Layout is every pool's vdevs and devices with their state and errors. +func (m *Machine) Layout(ctx context.Context) (any, error) { + st, err := m.Status(ctx) + if err != nil { + return notHere(err) + } + a := LayoutAnswer{Applicable: true, Pools: st, Said: []string{}} + for _, p := range st { + a.Said = append(a.Said, fmt.Sprintf("pool %s: %s", p.Name, p.State)) + walk(p.Root, func(d *Device, depth int) { + if d.Kind == "pool" { + return + } + line := strings.Repeat(" ", depth) + d.Name + if d.Kind == "section" { + a.Said = append(a.Said, line+" ("+d.Role+")") + return + } + line += " " + d.State + if d.Errors() > 0 { + line += fmt.Sprintf(" — ERRORS read %d, write %d, checksum %d", d.Read, d.Write, d.Checksum) + } + if d.Note != "" { + line += " " + d.Note + } + a.Said = append(a.Said, line) + }, 0) + } + return a, nil +} + +func walk(d *Device, f func(*Device, int), depth int) { + if d == nil { + return + } + f(d, depth) + for _, c := range d.Children { + walk(c, f, depth+1) + } +} + +func sumErrors(d *Device) uint64 { + var n uint64 + walk(d, func(x *Device, _ int) { + if x.Kind != "section" { + n += x.Errors() + } + }, 0) + return n +} + +func statusOf(st []PoolStatus, name string) *PoolStatus { + for i := range st { + if st[i].Name == name { + return &st[i] + } + } + return nil +} diff --git a/modules/zfs/cmd/zfs-tools/properties.go b/modules/zfs/cmd/zfs-tools/properties.go new file mode 100644 index 00000000..06cf0ff7 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/properties.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "strings" +) + +// PropertiesArgv is every property of one dataset, the name validated against the list first. +func PropertiesArgv(dataset string) []string { + return []string{"get", "-Hp", "-o", "property,value,source", "all", "--", dataset} +} + +// Withheld are properties whose value the module never answers: where an encryption key is kept can +// itself be a secret (a URL with a token). ZFS never gives a key itself. +var Withheld = map[string]bool{"keylocation": true} + +// Property is one property, with where its value comes from. +type Property struct { + Name string `json:"name"` + Value string `json:"value"` + Source string `json:"source"` +} + +// ParseProperties reads `zfs get -Hp -o property,value,source`. +func ParseProperties(out string) ([]Property, error) { + var ps []Property + for _, line := range lines(out) { + f := strings.Split(line, "\t") + if len(f) != 3 { + return nil, fmt.Errorf("zfs get answered a line with %d columns, not 3: %q", len(f), line) + } + p := Property{Name: f[0], Value: f[1], Source: f[2]} + if Withheld[p.Name] && p.Value != "none" && p.Value != "-" { + p.Value = "(withheld: where the key is kept is not answered)" + } + ps = append(ps, p) + } + return ps, nil +} + +// Highlights are the properties zfs_properties says in lines first. +var Highlights = []string{"type", "mountpoint", "mounted", "compression", "compressratio", "recordsize", "volblocksize", "atime", + "relatime", "sync", "xattr", "acltype", "dedup", "copies", "checksum", "sharenfs", "sharesmb", "readonly", "canmount", + "encryption", "keystatus", "keyformat", "encryptionroot", "snapdir", "primarycache", "secondarycache", "logbias", + "special_small_blocks", "quota", "refquota", "reservation", "refreservation"} + +// PropertiesAnswer is what zfs_properties answers. +type PropertiesAnswer struct { + Applicable bool `json:"applicable"` + Dataset string `json:"dataset"` + Properties []Property `json:"properties"` + Said []string `json:"said"` +} + +// PropertiesOf is every property of one dataset that exists. +func (m *Machine) PropertiesOf(ctx context.Context, dataset string) (any, error) { + if dataset == "" { + return nil, fmt.Errorf("name the dataset; zfs_datasets lists them") + } + ds, err := m.Datasets(ctx) + if err != nil { + return notHere(err) + } + if find(ds, dataset) == nil { + return nil, fmt.Errorf("there is no dataset %q on this machine; zfs_datasets lists them", dataset) + } + out, err := m.ask(ctx, "zfs", PropertiesArgv(dataset)...) + if err != nil { + return nil, err + } + ps, err := ParseProperties(out) + if err != nil { + return nil, err + } + a := PropertiesAnswer{Applicable: true, Dataset: dataset, Properties: ps, Said: []string{}} + by := map[string]Property{} + for _, p := range ps { + by[p.Name] = p + } + for _, name := range Highlights { + p, ok := by[name] + if !ok || p.Value == "-" { + continue + } + line := fmt.Sprintf("%s = %s", name, p.Value) + if p.Source != "-" && p.Source != "" { + line += " (" + p.Source + ")" + } + a.Said = append(a.Said, line) + } + return a, nil +} diff --git a/modules/zfs/cmd/zfs-tools/scrub.go b/modules/zfs/cmd/zfs-tools/scrub.go new file mode 100644 index 00000000..7a41abf3 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/scrub.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// ScrubWarnAfter is how long after the last completed scrub the module warns: a month and a few days, so +// a monthly scrub that ran late is not yet a warning. +const ScrubWarnAfter = 35 * 24 * time.Hour + +// HistoryTimeout bounds reading a pool's history, which grows with the pool's life. +const HistoryTimeout = 60 * time.Second + +// HistoryArgv is one pool's history with ZFS's internal events, where each scan's start and end are. +func HistoryArgv(pool string) []string { return []string{"history", "-i", pool} } + +// Pass is one scan the history records: when it started and how it ended. +type Pass struct { + Kind string `json:"kind"` + Started *time.Time `json:"started,omitempty"` + Ended *time.Time `json:"ended,omitempty"` + // Outcome is finished or canceled. + Outcome string `json:"outcome"` + Errors *int `json:"errors,omitempty"` +} + +var historyScan = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2}) \[txg:\d+\] scan (setup|done|cancelled|canceled|aborted)\b(.*)$`) +var historyFunc = regexp.MustCompile(`\bfunc=(\d+)`) +var historyErrors = regexp.MustCompile(`\berrors=(\d+)`) + +// ParseHistory reads `zpool history -i` for the last finished scrub and the last finished resilver. ZFS +// writes `scan setup … func=1` for a scrub and `func=2` for a resilver, then `scan done … errors=N` or +// `scan cancelled`. +func ParseHistory(out string) (scrub, resilver *Pass) { + var open *Pass + for _, line := range lines(out) { + m := historyScan.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + t, err := time.ParseInLocation("2006-01-02.15:04:05", m[1], time.Local) + if err != nil { + continue + } + switch m[2] { + case "setup": + kind := "scan" + if f := historyFunc.FindStringSubmatch(m[3]); f != nil { + switch f[1] { + case "1": + kind = "scrub" + case "2": + kind = "resilver" + } + } + open = &Pass{Kind: kind, Started: &t} + default: + if open == nil { + continue + } + open.Ended = &t + open.Outcome = "finished" + if m[2] != "done" { + open.Outcome = "canceled" + } + if e := historyErrors.FindStringSubmatch(m[3]); e != nil { + n, _ := strconv.Atoi(e[1]) + open.Errors = &n + } + if open.Outcome == "finished" { + switch open.Kind { + case "scrub": + scrub = open + case "resilver": + resilver = open + } + } + open = nil + } + } + return scrub, resilver +} + +// ScrubRow is one pool's scans. +type ScrubRow struct { + Pool string `json:"pool"` + // Last is the pool's last scan as status says it: ZFS keeps only the latest, a scrub's or a resilver's. + Last Scan `json:"last_scan"` + // LastScrub and LastResilver are the last finished of each, from the history when it was read. + LastScrub *Pass `json:"last_scrub,omitempty"` + LastResilver *Pass `json:"last_resilver,omitempty"` + DaysSince *float64 `json:"days_since_last_scrub,omitempty"` + Warning string `json:"warning,omitempty"` + Note string `json:"note,omitempty"` + Said string `json:"said"` +} + +// ScrubAnswer is what zfs_scrub answers. +type ScrubAnswer struct { + Applicable bool `json:"applicable"` + Pools []ScrubRow `json:"pools"` + Said []string `json:"said"` +} + +// LastScrubEnded is when the pool's last scrub finished, as status shows it; nil when status's last scan +// is not a finished scrub. +func LastScrubEnded(s Scan) *time.Time { + if s.Kind == "scrub" && s.State == "finished" { + return s.Ended + } + return nil +} + +// Scrubs are every pool's last scrub and resilver. With history, each pool's history is read too (as +// root), for the scan status no longer shows. +func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) { + st, err := m.Status(ctx) + if err != nil { + return notHere(err) + } + now := m.Now() + a := ScrubAnswer{Applicable: true, Pools: []ScrubRow{}, Said: []string{}} + for _, p := range st { + r := ScrubRow{Pool: p.Name, Last: p.Scan} + ended := LastScrubEnded(p.Scan) + if p.Scan.Kind == "resilver" && p.Scan.State == "finished" && p.Scan.Ended != nil { + r.LastResilver = &Pass{Kind: "resilver", Started: p.Scan.Started, Ended: p.Scan.Ended, Outcome: "finished", Errors: p.Scan.Errors} + } + if ended != nil { + r.LastScrub = &Pass{Kind: "scrub", Started: p.Scan.Started, Ended: ended, Outcome: "finished", Errors: p.Scan.Errors} + } + if history { + hctx, cancel := context.WithTimeout(ctx, HistoryTimeout) + out, err := m.AsRoot(hctx, "zpool", HistoryArgv(p.Name)...) + cancel() + if err != nil { + r.Note = "the history could not be read: " + err.Error() + } else { + scrub, resilver := ParseHistory(out.Out) + if scrub != nil && (r.LastScrub == nil || scrub.Ended.After(*r.LastScrub.Ended)) { + r.LastScrub = scrub + } + if resilver != nil && (r.LastResilver == nil || resilver.Ended.After(*r.LastResilver.Ended)) { + r.LastResilver = resilver + } + } + } else if r.LastScrub == nil { + r.Note = "status shows only the last scan, which is not a finished scrub; ask with history=true for the last scrub the pool's history records" + } + if r.LastScrub != nil && r.LastScrub.Ended != nil { + d := round1(now.Sub(*r.LastScrub.Ended).Hours() / 24) + r.DaysSince = &d + if now.Sub(*r.LastScrub.Ended) > ScrubWarnAfter { + r.Warning = fmt.Sprintf("the last scrub of %s finished %.0f days ago, more than %d", p.Name, d, int(ScrubWarnAfter.Hours()/24)) + } + } else if history { + r.Warning = fmt.Sprintf("no finished scrub of %s is on record", p.Name) + } + r.Said = scrubSaid(r) + a.Pools = append(a.Pools, r) + a.Said = append(a.Said, r.Said) + } + return a, nil +} + +func scrubSaid(r ScrubRow) string { + s := r.Last + var line string + switch s.State { + case "none": + line = fmt.Sprintf("%s: no scan was ever asked", r.Pool) + case "running", "paused": + line = fmt.Sprintf("%s: a %s is %s", r.Pool, s.Kind, s.State) + if s.Started != nil { + line += " since " + s.Started.Local().Format("2006-01-02 15:04") + } + if s.Done != nil { + line += fmt.Sprintf(", %.2f %% done", *s.Done) + } + if s.ToGo != "" { + line += ", " + s.ToGo + " to go" + } + case "finished": + line = fmt.Sprintf("%s: the last scan was a %s, finished %s after %s, %s repaired, %d errors", r.Pool, s.Kind, + stamp(s.Ended), s.Took, s.Repaired, deref(s.Errors)) + case "canceled": + line = fmt.Sprintf("%s: the last %s was canceled %s", r.Pool, s.Kind, stamp(s.Ended)) + default: + line = fmt.Sprintf("%s: scan: %s", r.Pool, firstLine(s.Said)) + } + if r.DaysSince != nil { + line += fmt.Sprintf("; last finished scrub %.1f days ago", *r.DaysSince) + } + if r.Warning != "" { + line += " — WARNING: " + r.Warning + } + return line +} + +func stamp(t *time.Time) string { + if t == nil { + return "at a time status did not say" + } + return t.Local().Format("2006-01-02 15:04") +} + +func deref(n *int) int { + if n == nil { + return 0 + } + return *n +} diff --git a/modules/zfs/cmd/zfs-tools/snapshot.go b/modules/zfs/cmd/zfs-tools/snapshot.go new file mode 100644 index 00000000..0b57cf13 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshot.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "time" +) + +// SnapshotEvent is the event each attempted snapshot is published as (module.json's emits): the record +// of the act, beside the pool's own history, which ZFS writes for every snapshot. +const SnapshotEvent = "snapshot.taken" + +// The one act of this module: a snapshot of one dataset, by a person, with why. It adds a snapshot and +// changes nothing else: no destroy, rollback, set or send exists in this module (novox/hq ADR 0051, the +// operator's data). ZFS takes a snapshot atomically, and with -r every child's at the same instant. + +// snapName is what a snapshot's own name may hold: letters, digits, dot, underscore and hyphen. +var snapName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +// MaxName is ZFS's limit on a whole name, dataset@snapshot (ZFS_MAX_DATASET_NAME_LEN less its end). +const MaxName = 255 + +// MaxWhy bounds the reason kept with the act. +const MaxWhy = 500 + +// SnapshotArgv is the command for one snapshot: fixed words, the dataset@name after `--`. +func SnapshotArgv(dataset, name string, recursive bool) []string { + argv := []string{"snapshot"} + if recursive { + argv = append(argv, "-r") + } + return append(argv, "--", dataset+"@"+name) +} + +// Made is one snapshot as `zfs list` shows it after the act. +type Made struct { + Name string `json:"name"` + Created string `json:"created"` + Used uint64 `json:"used_bytes"` + Referenced uint64 `json:"referenced_bytes"` +} + +// SnapshotAnswer is what zfs_snapshot answers: what the list shows after the act, not what was run. +type SnapshotAnswer struct { + Done bool `json:"done"` + Dataset string `json:"dataset"` + Name string `json:"snapshot"` + Recursive bool `json:"recursive"` + Why string `json:"why"` + Ran []string `json:"ran"` + Made []Made `json:"made"` + // Others are the snapshots that came or went on the machine during the act and were not this act's — + // another schedule's, say. Empty when nothing else changed. + Others []string `json:"others_changed"` + Recorded bool `json:"recorded"` + Record string `json:"record"` + Said string `json:"said"` +} + +// TakeSnapshot takes dataset@name. It refuses an unknown dataset, a name with anything but +// [A-Za-z0-9._-], an empty why, and a snapshot that already exists (for -r, on any child too). It +// answers what `zfs list` shows afterwards. +func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why string, recursive bool) (SnapshotAnswer, error) { + a := SnapshotAnswer{Dataset: dataset, Name: name, Recursive: recursive, Why: why, Made: []Made{}, Others: []string{}} + switch { + case dataset == "": + return a, fmt.Errorf("name the dataset; zfs_datasets lists them") + case strings.ContainsAny(dataset, "@#"): + return a, fmt.Errorf("%q is not a dataset: name the filesystem or volume, and the snapshot's name in name", dataset) + case name == "": + return a, fmt.Errorf("name the snapshot") + case !snapName.MatchString(name): + return a, fmt.Errorf("the snapshot's name %q may hold only letters, digits, '.', '_' and '-'", name) + case len(dataset)+1+len(name) > MaxName: + return a, fmt.Errorf("%s@%s is longer than ZFS's %d characters", dataset, name, MaxName) + case why == "": + return a, fmt.Errorf("say why: a snapshot is a person's act, and the why is kept with it") + case strings.ContainsAny(why, "\n\r"): + return a, fmt.Errorf("say why in one line") + case len(why) > MaxWhy: + return a, fmt.Errorf("say why in at most %d characters", MaxWhy) + } + ds, err := m.Datasets(ctx) + if err != nil { + return a, err + } + if find(ds, dataset) == nil { + return a, fmt.Errorf("there is no dataset %q on this machine; zfs_datasets lists them", dataset) + } + before, err := m.Snapshots(ctx) + if err != nil { + return a, err + } + covered := []string{dataset} + if recursive { + for _, d := range ds { + if strings.HasPrefix(d.Name, dataset+"/") { + covered = append(covered, d.Name) + } + } + } + had := names(before) + for _, d := range covered { + if had[d+"@"+name] { + return a, fmt.Errorf("the snapshot %s@%s already exists; choose another name", d, name) + } + } + + program, argv := m.Escalated("zfs", SnapshotArgv(dataset, name, recursive)...) + a.Ran = append([]string{program}, argv...) + _, runErr := m.AsRoot(ctx, "zfs", SnapshotArgv(dataset, name, recursive)...) + + // What is there now, whatever the command said. + after, listErr := m.Snapshots(ctx) + now := names(after) + expected := map[string]bool{} + for _, d := range covered { + expected[d+"@"+name] = true + } + for _, s := range after { + if expected[s.Name] { + a.Made = append(a.Made, Made{Name: s.Name, Created: s.Created.Local().Format(time.RFC3339), Used: s.Used, Referenced: s.Referenced}) + } + } + if listErr == nil { + for n := range now { + if !had[n] && !expected[n] { + a.Others = append(a.Others, "appeared: "+n) + } + } + for n := range had { + if !now[n] { + a.Others = append(a.Others, "went: "+n) + } + } + sort.Strings(a.Others) + } + a.Done = runErr == nil && listErr == nil && len(a.Made) == len(covered) + + switch { + case runErr != nil && len(a.Made) == 0: + a.Said = fmt.Sprintf("NOT DONE: %v. No snapshot %s@%s is on the machine.", runErr, dataset, name) + case runErr != nil: + a.Said = fmt.Sprintf("zfs snapshot failed (%v), and yet the list shows %s.", runErr, madeNames(a.Made)) + case listErr != nil: + a.Said = fmt.Sprintf("zfs snapshot said it succeeded, and the snapshots could not be listed again to see it: %v.", listErr) + case len(a.Made) != len(covered): + a.Said = fmt.Sprintf("zfs snapshot said it succeeded, and the list shows %d of the %d snapshots it should have made: %s.", + len(a.Made), len(covered), madeNames(a.Made)) + default: + a.Said = fmt.Sprintf("Made %s, created %s. Why: %s.", madeNames(a.Made), a.Made[0].Created, strings.TrimSuffix(why, ".")) + if len(a.Others) == 0 { + a.Said += " No other snapshot came or went meanwhile." + } else { + a.Said += fmt.Sprintf(" Meanwhile, not by this act: %s.", strings.Join(a.Others, ", ")) + } + } + + a.Recorded, a.Record = m.record(a) + if !a.Recorded { + a.Said += " NOT RECORDED on the bus: " + a.Record + "; the pool's own history (zpool history) has the command." + } + if !a.Done { + // Failing is loud: a snapshot that is not there is never answered as made. + return a, fmt.Errorf("%s", a.Said) + } + return a, nil +} + +// record publishes the act as this module's event: the names, the reason and the outcome, nothing else. +func (m *Machine) record(a SnapshotAnswer) (bool, string) { + if m.Emit == nil { + return false, "this process publishes no events" + } + created := "" + if len(a.Made) > 0 { + created = a.Made[0].Created + } + err := m.Emit(SnapshotEvent, map[string]any{"dataset": a.Dataset, "snapshot": a.Name, "recursive": a.Recursive, + "why": a.Why, "done": a.Done, "made": madeNames(a.Made), "created": created, "at": m.Now().UTC().Format(time.RFC3339)}) + if err != nil { + return false, err.Error() + } + return true, "published as the event " + SnapshotEvent +} + +func names(ss []Snapshot) map[string]bool { + out := map[string]bool{} + for _, s := range ss { + out[s.Name] = true + } + return out +} + +func madeNames(ms []Made) string { + if len(ms) == 0 { + return "nothing" + } + var ns []string + for _, x := range ms { + ns = append(ns, x.Name) + } + return strings.Join(ns, ", ") +} diff --git a/modules/zfs/cmd/zfs-tools/snapshot_test.go b/modules/zfs/cmd/zfs-tools/snapshot_test.go new file mode 100644 index 00000000..645d5fe6 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshot_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +const takeLine = "sudo -n zfs snapshot -- tank/media@before-move" + +// snapshotting makes the fake's snapshot command add a snapshot to what the list then answers, with a +// creation time of its own: the answer must carry the list's time, not the clock's. +func snapshotting(f *fake, cmd string, added ...string) { + f.on(cmd, "") + f.after[cmd] = func(f *fake) { + key := line("zfs", SnapshotsArgv()) + r := f.answers[key] + for _, a := range added { + r.Out += a + "\n" + } + f.answers[key] = r + } +} + +func TestTheSnapshotCommandIsFixedWordsThroughSudoWithTheNameAfterDashDash(t *testing.T) { + m := &Machine{UID: 1000} + p, argv := m.Escalated("zfs", SnapshotArgv("tank/media", "before-move", false)...) + if got := p + " " + strings.Join(argv, " "); got != takeLine { + t.Errorf("ran %q", got) + } + p, argv = m.Escalated("zfs", SnapshotArgv("tank", "x", true)...) + if got := p + " " + strings.Join(argv, " "); got != "sudo -n zfs snapshot -r -- tank@x" { + t.Errorf("recursive: %q", got) + } + root := &Machine{UID: 0} + if p, argv := root.Escalated("zfs", SnapshotArgv("tank", "x", false)...); p != "zfs" || argv[0] != "snapshot" { + t.Errorf("as root, no sudo: %s %v", p, argv) + } +} + +func TestASnapshotIsAnsweredFromTheListAfterwardsAndRecorded(t *testing.T) { + f := withZFS(t) + snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t26938034880512") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "before moving the library", false) + if err != nil { + t.Fatal(err) + } + if !a.Done || len(a.Made) != 1 || a.Made[0].Name != "tank/media@before-move" || a.Made[0].Referenced != 26938034880512 { + t.Fatalf("answer: %+v", a) + } + if a.Made[0].Created != time.Unix(1791400000, 0).Local().Format(time.RFC3339) || a.Made[0].Created == f.now.Format(time.RFC3339) { + t.Errorf("the creation time is the list's: %s", a.Made[0].Created) + } + if len(a.Others) != 0 || !strings.Contains(a.Said, "No other snapshot came or went") || !strings.Contains(a.Said, "Why: before moving the library") { + t.Errorf("said: %s / %v", a.Said, a.Others) + } + if strings.Join(a.Ran, " ") != takeLine || len(ran(f, "sudo")) != 1 { + t.Errorf("ran: %v; every call: %v", a.Ran, f.calls) + } + if len(f.emitted) != 1 || f.emitted[0]["_type"] != SnapshotEvent || f.emitted[0]["why"] != "before moving the library" || + f.emitted[0]["done"] != true || f.emitted[0]["made"] != "tank/media@before-move" || !a.Recorded { + t.Errorf("recorded: %+v", f.emitted) + } +} + +func TestASnapshotTheListDoesNotShowIsNotAnsweredAsMade(t *testing.T) { + f := withZFS(t) + f.on(takeLine, "") // zfs exits 0, and the list does not change + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", false) + if err == nil || a.Done || !strings.Contains(err.Error(), "the list shows 0 of the 1") { + t.Fatalf("err %v, answer %+v", err, a) + } + if len(f.emitted) != 1 || f.emitted[0]["done"] != false { + t.Errorf("the failed act is recorded too: %+v", f.emitted) + } +} + +func TestARefusedSudoIsSaidAndRecorded(t *testing.T) { + f := withZFS(t) + f.answers[takeLine] = Ran{Code: 1, Err: "sudo: a password is required"} + _, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", false) + if err == nil || !strings.Contains(err.Error(), "NOT DONE") || !strings.Contains(err.Error(), "sudo refused the operator account") { + t.Fatalf("err: %v", err) + } + if len(f.emitted) != 1 || f.emitted[0]["done"] != false { + t.Errorf("recorded: %+v", f.emitted) + } +} + +func TestAnUnrecordedActSaysSoLoudly(t *testing.T) { + f := withZFS(t) + snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t1") + f.emitErr = errors.New("the bus did not accept it") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", false) + if err != nil { + t.Fatal(err) + } + if a.Recorded || !strings.Contains(a.Said, "NOT RECORDED on the bus: the bus did not accept it") { + t.Errorf("said: %s", a.Said) + } +} + +func TestAnotherSnapshotMeanwhileIsSaidNotHidden(t *testing.T) { + f := withZFS(t) + snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t1", "tank/downloads@auto-1200\t1791547201\t0\t1") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", false) + if err != nil { + t.Fatal(err) + } + if len(a.Others) != 1 || a.Others[0] != "appeared: tank/downloads@auto-1200" || !strings.Contains(a.Said, "not by this act") { + t.Errorf("others: %v; said %s", a.Others, a.Said) + } +} + +func TestEveryRefusalRunsNoSnapshotCommand(t *testing.T) { + cases := []struct{ dataset, name, why, says string }{ + {"tank/nope", "x", "why", "no dataset"}, + {"", "x", "why", "name the dataset"}, + {"tank/media@x", "y", "why", "is not a dataset"}, + {"tank/media", "", "why", "name the snapshot"}, + {"tank/media", "a b", "why", "may hold only"}, + {"tank/media", "x;rm -rf /", "why", "may hold only"}, + {"tank/media", "$(reboot)", "why", "may hold only"}, + {"tank/media", "a/b", "why", "may hold only"}, + {"tank/media", "a@b", "why", "may hold only"}, + {"tank/media", "-r", "why", ""}, // a name may start with '-': it follows -- and the dataset's @ + {"tank/media", strings.Repeat("x", 250), "why", "longer than"}, + {"tank/media", "before-rename", "why", "already exists"}, + {"tank/media", "x", "", "say why"}, + {"tank/media", "x", "one\ntwo", "one line"}, + } + for _, c := range cases { + f := withZFS(t) + _, err := f.machine().TakeSnapshot(context.Background(), c.dataset, c.name, c.why, false) + if c.says == "" { + if len(ran(f, "sudo -n zfs snapshot -- tank/media@-r")) != 1 { + t.Errorf("%q: the name follows -- and runs as a name: %v", c.name, f.calls) + } + continue + } + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%q@%q why %q: %v", c.dataset, c.name, c.why, err) + } + if n := len(ran(f, "sudo")); n != 0 { + t.Errorf("%q@%q: a refusal ran %v", c.dataset, c.name, f.calls) + } + } +} + +func TestRecursiveIsRefusedWhenAChildHasTheNameAlready(t *testing.T) { + f := withZFS(t) + _, err := f.machine().TakeSnapshot(context.Background(), "tank", "weekly", "why", true) + if err == nil || !strings.Contains(err.Error(), "tank/downloads@weekly already exists") { + t.Fatalf("err: %v", err) + } + // Without recursive, only tank's own name counts. + f = withZFS(t) + snapshotting(f, "sudo -n zfs snapshot -- tank@weekly", "tank@weekly\t1791400000\t0\t1") + if a, err := f.machine().TakeSnapshot(context.Background(), "tank", "weekly", "why", false); err != nil || len(a.Made) != 1 { + t.Errorf("tank@weekly alone: %v %+v", err, a) + } +} + +func TestRecursiveAnswersEveryChildsSnapshot(t *testing.T) { + f := withZFS(t) + snapshotting(f, "sudo -n zfs snapshot -r -- tank@all", "tank@all\t1\t0\t1", "tank/media@all\t1\t0\t1", + "tank/downloads@all\t1\t0\t1", "tank/vm@all\t1\t0\t1") + a, err := f.machine().TakeSnapshot(context.Background(), "tank", "all", "why", true) + if err != nil || len(a.Made) != 4 || !a.Recursive { + t.Fatalf("%v %+v", err, a) + } +} + +func TestAMachineWithoutZFSSaysTheModuleDoesNotApply(t *testing.T) { + f := newFake(t) // every command: executable file not found + m := f.machine() + for _, tool := range Tools(m) { + if tool.Name == "zfs_health" { + continue + } + args := map[string]any{"dataset": "tank", "name": "x", "why": "y"} + got, err := tool.Run(args) + if err != nil { + t.Errorf("%s: %v", tool.Name, err) + continue + } + na, ok := got.(NotApplicable) + if !ok || na.Applicable || !strings.Contains(na.Said, "does not apply here") { + t.Errorf("%s: %+v", tool.Name, got) + } + } + h := m.Health(context.Background()) + if h.Healthy || !strings.Contains(h.Why, "not installed") || !strings.Contains(h.Why, "Unassign") { + t.Errorf("health: %+v", h) + } + if n := len(ran(f, "sudo -n zfs snapshot")); n != 0 { + t.Errorf("no snapshot is tried: %v", f.calls) + } +} + +func TestAMissingKernelModuleIsNotApplicableToo(t *testing.T) { + r := Ran{Code: 1, Err: "The ZFS modules cannot be auto-loaded.\nTry running 'modprobe zfs' as root to manually load them."} + if why := absent(r); !strings.Contains(why, "kernel module is not loaded") { + t.Errorf("absent: %q", why) + } +} diff --git a/modules/zfs/cmd/zfs-tools/testdata/arcstats-nol2.txt b/modules/zfs/cmd/zfs-tools/testdata/arcstats-nol2.txt new file mode 100644 index 00000000..793144d8 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/arcstats-nol2.txt @@ -0,0 +1,12 @@ +13 1 0x01 147 39984 7071415536 284772115112591 +name type data +hits 4 90 +misses 4 10 +c 4 1073741824 +c_min 4 1051456512 +c_max 4 2147483648 +size 4 1000000000 +l2_hits 4 0 +l2_misses 4 0 +l2_size 4 0 +l2_asize 4 0 diff --git a/modules/zfs/cmd/zfs-tools/testdata/arcstats.txt b/modules/zfs/cmd/zfs-tools/testdata/arcstats.txt new file mode 100644 index 00000000..8817a7d6 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/arcstats.txt @@ -0,0 +1,17 @@ +13 1 0x01 147 39984 7071415536 284772115112591 +name type data +hits 4 908234567 +iohits 4 1234567 +misses 4 95765433 +demand_data_hits 4 500000000 +demand_data_misses 4 50000000 +demand_metadata_hits 4 300000000 +demand_metadata_misses 4 10000000 +c 4 16823304192 +c_min 4 1051456512 +c_max 4 33646608384 +size 4 16798000128 +l2_hits 4 1200000 +l2_misses 4 3600000 +l2_size 4 412316860416 +l2_asize 4 398000000000 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zfs-get.txt b/modules/zfs/cmd/zfs-tools/testdata/zfs-get.txt new file mode 100644 index 00000000..995771b8 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zfs-get.txt @@ -0,0 +1,16 @@ +type filesystem - +creation 1700000000 - +used 27487790694400 - +mountpoint /storage/media inherited from tank +compression lz4 inherited from tank +compressratio 1.00 - +recordsize 1048576 local +atime off inherited from tank +sharenfs rw=@192.0.2.0/24 local +encryption aes-256-gcm - +keylocation https://keys.example/k?token=s3cret local +keyformat passphrase - +keystatus available - +encryptionroot tank/media - +quota 0 default +tank:purpose the library local diff --git a/modules/zfs/cmd/zfs-tools/testdata/zfs-list-snapshots.txt b/modules/zfs/cmd/zfs-tools/testdata/zfs-list-snapshots.txt new file mode 100644 index 00000000..ca0e030e --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zfs-list-snapshots.txt @@ -0,0 +1,4 @@ +tank/media@2026-08-01 1785542400 274877906944 26388279066624 +tank/media@2026-09-01 1788220800 137438953472 26800000000000 +tank/media@before-rename 1791100800 1048576 26938034880512 +tank/downloads@weekly 1790496000 0 1099511627776 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zfs-list.txt b/modules/zfs/cmd/zfs-tools/testdata/zfs-list.txt new file mode 100644 index 00000000..858a3150 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zfs-list.txt @@ -0,0 +1,4 @@ +tank filesystem /storage 28591834521600 11203942195200 196608 1.01 off 0 196608 28591834324992 0 0 0 0 0 28870035210240 42496 lz4 +tank/media filesystem /storage/media 27487790694400 11203942195200 26938034880512 1.00 rw=@192.0.2.0/24 549755813888 26938034880512 0 0 0 0 0 0 27500000000000 26950000000000 lz4 +tank/downloads filesystem /storage/downloads 1099511627776 11203942195200 1099511627776 1.02 off 0 1099511627776 0 0 2199023255552 0 0 0 1121501860331 1121501860331 lz4 +tank/vm volume - 4398046511 11208340241711 2147483648 1.85 - 0 2147483648 0 2250562863 - - 0 4398046511 3972844748 3972844748 zstd diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-events.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-events.txt new file mode 100644 index 00000000..eaacbcf1 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-events.txt @@ -0,0 +1,40 @@ +TIME CLASS +Oct 8 2026 20:58:02.112233445 sysevent.fs.zfs.history_event + version = 0x0 + class = "sysevent.fs.zfs.history_event" + pool = "tank" + history_internal_str = "func=1 mintxg=0" + eid = 0x10 + +Oct 8 2026 21:01:55.889900112 ereport.fs.zfs.io + class = "ereport.fs.zfs.io" + ena = 0x5e1f2b3c4d500001 + pool = "tank" + pool_guid = 0x1234567890abcdef + vdev_path = "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_D7JA0002-part1" + vdev_type = "disk" + zio_err = 0x5 + eid = 0x11 + +Oct 8 2026 21:02:01.000000001 resource.fs.zfs.statechange + version = 0x0 + class = "resource.fs.zfs.statechange" + pool = "tank" + vdev_path = "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_D7JA0002-part1" + vdev_state = 0x5 + vdev_laststate = 0x7 + eid = 0x12 + +Oct 8 2026 21:02:01.500000000 sysevent.fs.zfs.config_sync + version = 0x0 + class = "sysevent.fs.zfs.config_sync" + pool = "tank" + eid = 0x13 + +Oct 8 2026 21:02:17.000000000 sysevent.fs.zfs.resilver_start + version = 0x0 + class = "sysevent.fs.zfs.resilver_start" + pool = "tank" + resilver_type = "healing" + eid = 0x14 + diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-history.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-history.txt new file mode 100644 index 00000000..95f80c4d --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-history.txt @@ -0,0 +1,9 @@ +History for 'tank': +2026-07-12.00:24:01 [txg:4410021] scan setup tank (552) func=1 mintxg=0 maxtxg=4410021 +2026-07-13.02:30:00 [txg:4419020] scan done tank (552) errors=0 +2026-08-02.10:00:00 zfs snapshot tank/media@2026-08-01 +2026-08-20.13:12:40 [txg:4621100] scan setup tank (552) func=2 mintxg=3 maxtxg=4621100 +2026-08-20.19:40:02 [txg:4623199] scan done tank (552) errors=0 +2026-09-13.00:24:01 [txg:4701131] scan setup tank (552) func=1 mintxg=0 maxtxg=4701131 +2026-09-13.03:00:00 [txg:4702000] scan cancelled tank (552) errors=0 +2026-10-08.21:02:17 [txg:4801131] scan setup tank (552) func=2 mintxg=3 maxtxg=4801131 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-iostat.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-iostat.txt new file mode 100644 index 00000000..279b2982 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-iostat.txt @@ -0,0 +1,12 @@ +tank 28797591658086 19184805568512 212 14 221249536 1048576 +raidz2-0 28797591658086 19184805568512 212 14 221249536 1048576 +ata-WDC_WD120EFBX-68B0EN0_D7JA0001 - - 35 2 36874922 174762 +ata-WDC_WD120EFBX-68B0EN0_D7JA0002 - - 36 2 36874922 174762 +logs - - - - - - +mirror-1 12582912 16093544448 0 0 0 0 +nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100001-part1 - - 0 0 0 0 +nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100002-part1 - - 0 0 0 0 +cache - - - - - - +nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100001-part2 480000000000 520000000000 40 1 5242880 131072 +rpool 85899345920 412316860416 3 25 98304 1310720 +nvme-Samsung_SSD_970_EVO_Plus_500GB_S4EVNX0R200001-part2 85899345920 412316860416 3 25 98304 1310720 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-list-full.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-list-full.txt new file mode 100644 index 00000000..6721a1fb --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-list-full.txt @@ -0,0 +1 @@ +tank 47982397226598 43664981476204 4317415750394 91 27 DEGRADED 1.00 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-list.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-list.txt new file mode 100644 index 00000000..7261d95c --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-list.txt @@ -0,0 +1,2 @@ +tank 47982397226598 28797591658086 19184805568512 60 3 ONLINE 1.00 +rpool 498216206336 85899345920 412316860416 17 - ONLINE 1.00 diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-status-degraded.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-status-degraded.txt new file mode 100644 index 00000000..263fa63d --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-status-degraded.txt @@ -0,0 +1,23 @@ + pool: tank + state: DEGRADED +status: One or more devices is currently being resilvered. The pool will + continue to function, possibly in a degraded state. +action: Wait for the resilver to complete. + scan: resilver in progress since Thu Oct 8 21:02:17 2026 + 9876543210123 / 28797591658086 scanned at 1101004800/s, 6543210987654 / 28797591658086 issued at 734003200/s + 1088516226 resilvered, 22.72% done, 08:25:13 to go +config: + + NAME STATE READ WRITE CKSUM + tank DEGRADED 0 0 0 + raidz2-0 DEGRADED 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0001 ONLINE 0 0 0 + spare-1 DEGRADED 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0002 FAULTED 12 107 3 too many errors + ata-WDC_WD120EFBX-68B0EN0_D7JA0009 ONLINE 0 0 0 (resilvering) + ata-WDC_WD120EFBX-68B0EN0_D7JA0003 ONLINE 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0004 ONLINE 0 0 0 + spares + ata-WDC_WD120EFBX-68B0EN0_D7JA0009 INUSE currently in use + +errors: 2 data errors, use '-v' for a list diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-status-scrubbing.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-status-scrubbing.txt new file mode 100644 index 00000000..d19c9f35 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-status-scrubbing.txt @@ -0,0 +1,14 @@ + pool: tank + state: ONLINE + scan: scrub in progress since Sun Oct 4 00:24:01 2026 + 13523942719488 / 28797591658086 scanned at 1298300000/s, 8400000000000 / 28797591658086 issued at 830000000/s + 0 repaired, 29.17% done, 06:49:34 to go +config: + + NAME STATE READ WRITE CKSUM + tank ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-ST16000NM001G-2KK103_ZL200001 ONLINE 0 0 0 + ata-ST16000NM001G-2KK103_ZL200002 ONLINE 0 0 0 + +errors: No known data errors diff --git a/modules/zfs/cmd/zfs-tools/testdata/zpool-status.txt b/modules/zfs/cmd/zfs-tools/testdata/zpool-status.txt new file mode 100644 index 00000000..ac21bf87 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/testdata/zpool-status.txt @@ -0,0 +1,38 @@ + pool: rpool + state: ONLINE +status: Some supported and requested features are not enabled on the pool. + The pool can still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(7) for details. + scan: scrub repaired 0 in 00:04:11 with 0 errors on Sun Jul 12 00:28:12 2026 +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + nvme-Samsung_SSD_970_EVO_Plus_500GB_S4EVNX0R200001-part2 ONLINE 0 0 0 + +errors: No known data errors + + pool: tank + state: ONLINE + scan: scrub repaired 0 in 1 days 02:10:05 with 0 errors on Mon Sep 14 02:34:06 2026 +config: + + NAME STATE READ WRITE CKSUM + tank ONLINE 0 0 0 + raidz2-0 ONLINE 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0001 ONLINE 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0002 ONLINE 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0003 ONLINE 0 0 0 + ata-WDC_WD120EFBX-68B0EN0_D7JA0004 ONLINE 0 0 0 + logs + mirror-1 ONLINE 0 0 0 + nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100001-part1 ONLINE 0 0 0 + nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100002-part1 ONLINE 0 0 0 + cache + nvme-Samsung_SSD_980_PRO_1TB_S5P2NG0R100001-part2 ONLINE 0 0 0 + spares + ata-WDC_WD120EFBX-68B0EN0_D7JA0009 AVAIL + +errors: No known data errors diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go new file mode 100644 index 00000000..80a9e57f --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -0,0 +1,223 @@ +package main + +import ( + "context" + "fmt" + "math" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Tools are the module's tools over one machine. Every one but zfs_snapshot only reads; zfs_snapshot only +// adds a snapshot. Nothing here destroys, rolls back, sets a property or sends: those change the +// operator's data and need a decision of their own. +func Tools(m *Machine) []stdio.Tool { + ctx := context.Background() + dataset := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + return []stdio.Tool{ + { + Name: "zfs_datasets", + Description: "Every ZFS filesystem and volume on this machine: name, mountpoint, used, available and referenced " + + "(bytes), compression ratio, sharenfs, how many snapshots it has and the space they hold together " + + "(usedbysnapshots: what destroying all of them would free), each in a line under `said`. A machine " + + "without ZFS answers applicable false, with why. Replaces zfs list. (r)", + Run: func(map[string]any) (any, error) { return m.DatasetsNow(ctx) }, + }, + { + Name: "zfs_snapshots", + Description: "The snapshots of one dataset (exactly that one, not its children), or of every dataset when none is " + + "named, oldest first: name, creation time, used (the space only that snapshot holds: what destroying it " + + "alone would free) and referenced. Replaces zfs list -t snapshot. (r)", + Input: map[string]any{"dataset": dataset("the dataset, as zfs_datasets names it (optional: every snapshot when left out)")}, + Run: func(a map[string]any) (any, error) { return m.SnapshotsOf(ctx, text(a, "dataset")) }, + }, + { + Name: "zfs_snapshot", + Description: "Take a snapshot dataset@name of one dataset, a person's act with why: refused for a dataset that is not " + + "on this machine, a name with anything but letters, digits, '.', '_' and '-', a snapshot that already exists, " + + "or no why. Only that dataset unless recursive is true, which snapshots every child at the same instant too. " + + "Runs zfs snapshot -- dataset@name through sudo -n, then answers what zfs list shows: the snapshot made " + + "and its creation time, and any other snapshot that came or went meanwhile. Recorded as the event " + + "snapshot.taken; ZFS keeps it in the pool's history too. It adds a snapshot and changes nothing else. " + + "Replaces zfs snapshot. (a)", + Input: map[string]any{ + "dataset": dataset("the dataset, as zfs_datasets names it"), + "name": map[string]any{"type": "string", "description": "the snapshot's name, after the @: letters, digits, '.', '_' and '-'"}, + "why": map[string]any{"type": "string", "description": "why, in one line; kept with the act"}, + "recursive": map[string]any{"type": "boolean", "description": "true to snapshot every child dataset too (default false)"}, + }, + Run: func(a map[string]any) (any, error) { + recursive, err := flag(a, "recursive") + if err != nil { + return nil, err + } + ans, err := m.TakeSnapshot(ctx, text(a, "dataset"), text(a, "name"), text(a, "why"), recursive) + if err != nil { + if _, absentHere := err.(errAbsent); absentHere { + return NotApplicable{Applicable: false, Said: err.Error()}, nil + } + return nil, err + } + return ans, nil + }, + }, + { + Name: "zfs_pools", + Description: "Every imported pool's health in a line: its state (ONLINE, DEGRADED, FAULTED, …) with status's own " + + "explanation and action, size, allocated and free (bytes), capacity and fragmentation %, the error count of " + + "its devices, its data errors and its last scan. Replaces zpool list and zpool status -x. (r)", + Run: func(map[string]any) (any, error) { return m.PoolsNow(ctx) }, + }, + { + Name: "zfs_space", + Description: "Where the space goes: each pool's size, allocated, free, capacity % and fragmentation %; then every " + + "dataset, biggest first, with what uses it (usedbydataset, usedbysnapshots, usedbychildren, " + + "usedbyrefreservation), its quota, refquota, reservation and refreservation where set, the compression and " + + "its ratio, the logical size against the physical and what compression saves. Replaces zfs list -o space " + + "and zpool list -v. (r)", + Run: func(map[string]any) (any, error) { return m.Space(ctx) }, + }, + { + Name: "zfs_pool_layout", + Description: "Each pool's layout as status gives it: its vdevs (mirror, raidz, draid) and their disks by the name the " + + "pool knows them by (by-id where it was imported so), each with its state and its read, write and checksum " + + "error counts and status's note (resilvering, was /dev/…); then the log, cache, special and dedup devices " + + "and the spares. Replaces zpool status and zpool status -v. (r)", + Run: func(map[string]any) (any, error) { return m.Layout(ctx) }, + }, + { + Name: "zfs_scrub", + Description: "Each pool's last scan, as status says it (ZFS keeps only the latest, a scrub's or a resilver's): when it " + + "started and ended, how long it took, what it repaired and the errors it found, or, while one runs, its " + + "progress and the time to go; and how many days since the last finished scrub, with a WARNING past 35. " + + "With history true, also each pool's history (zpool history -i, as root through sudo -n) for the last " + + "finished scrub and resilver status no longer shows; it can take some seconds on a pool with a long life. " + + "Replaces zpool history -i. (r)", + Input: map[string]any{"history": map[string]any{"type": "boolean", "description": "true to read each pool's history too (default false)"}}, + Run: func(a map[string]any) (any, error) { + h, err := flag(a, "history") + if err != nil { + return nil, err + } + return m.Scrubs(ctx, h) + }, + }, + { + Name: "zfs_io", + Description: "A sample of every pool's I/O over a few seconds (zpool iostat -v, the report since boot left out): " + + "read and write operations per second and bytes per second per pool, vdev and device, with each " + + "vdev's allocated and free. Replaces zpool iostat and zpool iostat -v. (r)", + Input: map[string]any{"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, 1 to 10)"}}, + Run: func(a map[string]any) (any, error) { + n, err := whole(a, "seconds", 3, 1, 10) + if err != nil { + return nil, err + } + return m.IO(ctx, n) + }, + }, + { + Name: "zfs_arc", + Description: "The ARC, ZFS's cache in memory, since the machine started, from /proc/spl/kstat/zfs/arcstats: its " + + "size, target, minimum and maximum (bytes), hits, misses and hit ratio %, the ratio for what was asked " + + "for alone (not read ahead), and the L2ARC on a cache device where there is one. Replaces arcstat, " + + "arc_summary and cat /proc/spl/kstat/zfs/arcstats. (r)", + Run: func(map[string]any) (any, error) { return m.Arc() }, + }, + { + Name: "zfs_events", + Description: "ZFS's recent events, newest first, as root through sudo -n: by default only errors (I/O, checksum and " + + "device errors) and changes of state (a device's state, a scrub or resilver started or finished, a pool " + + "imported), each with its time, class, pool, device and state; all true gives every event. ZFS keeps " + + "a bounded number since its module loaded. Replaces zpool events and zpool events -v. (r)", + Input: map[string]any{ + "all": map[string]any{"type": "boolean", "description": "true for every event, not only errors and changes of state (default false)"}, + "limit": map[string]any{"type": "integer", "description": "how many to show (default 50, at most 500)"}, + }, + Run: func(a map[string]any) (any, error) { + all, err := flag(a, "all") + if err != nil { + return nil, err + } + n, err := whole(a, "limit", 50, 1, 500) + if err != nil { + return nil, err + } + return m.Events(ctx, all, n) + }, + }, + { + Name: "zfs_properties", + Description: "Every property of one dataset with where its value comes from (local, default, inherited from …): " + + "compression, recordsize, atime, sync, xattr, sharenfs, sharesmb, mountpoint, readonly, encryption, " + + "keystatus, keyformat and encryptionroot and the rest, the usual ones said first in lines. Never a key: " + + "keylocation's value is withheld. Replaces zfs get all. (r)", + Input: map[string]any{"dataset": dataset("the dataset, as zfs_datasets names it")}, + Run: func(a map[string]any) (any, error) { return m.PropertiesOf(ctx, text(a, "dataset")) }, + }, + { + Name: "zfs_health", + Description: "Whether this machine's ZFS is healthy, as the module's health check (novox/hq ADR 0240): unhealthy " + + "when a pool is not ONLINE (with status's reason and action), a device has read, write or checksum " + + "errors or is not online, a pool reports data errors, a pool is more than 90 % full, no pool is " + + "imported, or ZFS is not on this machine; a pool more than 80 % full and a last scrub more than 35 " + + "days ago are warnings, which leave it healthy. With why, one sentence per finding. (r)", + Run: func(map[string]any) (any, error) { return m.Health(ctx), nil }, + }, + } +} + +func text(a map[string]any, key string) string { + s, _ := a[key].(string) + return strings.TrimSpace(s) +} + +// flag is a yes-or-no argument: true only when given as true; anything that is not a yes or a no is +// refused rather than read as no. +func flag(a map[string]any, key string) (bool, error) { + switch v := a[key].(type) { + case nil: + return false, nil + case bool: + return v, nil + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "true": + return true, nil + case "false", "": + return false, nil + } + return false, fmt.Errorf("%s is true or false, not %q", key, v) + } + return false, fmt.Errorf("%s is true or false", key) +} + +// whole is a whole-number argument, defaulted, refused outside least..most. +func whole(a map[string]any, key string, fallback, least, most int) (int, error) { + v, given := a[key] + if !given || v == nil { + return fallback, nil + } + var n int + switch x := v.(type) { + case float64: + if x != math.Trunc(x) { + return 0, fmt.Errorf("%s must be a whole number, not %v", key, x) + } + n = int(x) + case string: + i, err := strconv.Atoi(strings.TrimSpace(x)) + if err != nil { + return 0, fmt.Errorf("%s must be a whole number, not %q", key, x) + } + n = i + default: + return 0, fmt.Errorf("%s must be a whole number", key) + } + if n < least || n > most { + return 0, fmt.Errorf("%s must be from %d to %d, not %d", key, least, most, n) + } + return n, nil +} diff --git a/modules/zfs/cmd/zfs-tools/watch.go b/modules/zfs/cmd/zfs-tools/watch.go new file mode 100644 index 00000000..a6bedbb1 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/watch.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "fmt" + "io" + "time" +) + +// WatchEvery is how often the watch looks at the pools. +const WatchEvery = time.Minute + +// The module's one long-running process (`zfs-tools watch`), run by the node-engine. It is what the +// module's health is declared on (novox/hq ADR 0240: a tool check stands on a long-running resource): the +// node-engine judges the process alive, and asks zfs_health whether the pools are. The watch itself +// writes no file and changes nothing: each minute it reads the pools, and says in its journal when their +// health changes, so the journal holds when a pool went DEGRADED and when it came back. + +// Watch keeps the last verdict, to say only changes. +type Watch struct { + M *Machine + Out io.Writer + last string +} + +// Look reads the pools once and says the verdict when it changed. +func (w *Watch) Look(ctx context.Context) { + h := w.M.Health(ctx) + verdict := "healthy" + if !h.Healthy { + verdict = "UNHEALTHY" + } + line := verdict + ": " + h.Why + if line != w.last { + fmt.Fprintf(w.Out, "%s %s\n", w.M.Now().Format(time.RFC3339), line) + w.last = line + } +} + +// Run looks every WatchEvery until the context ends. +func (w *Watch) Run(ctx context.Context) { + w.Look(ctx) + t := time.NewTicker(WatchEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + w.Look(ctx) + } + } +} diff --git a/modules/zfs/go.mod b/modules/zfs/go.mod new file mode 100644 index 00000000..a48dc8b2 --- /dev/null +++ b/modules/zfs/go.mod @@ -0,0 +1,5 @@ +module zfstools + +go 1.22 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/zfs/go.sum b/modules/zfs/go.sum new file mode 100644 index 00000000..b474419e --- /dev/null +++ b/modules/zfs/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/zfs/module.json b/modules/zfs/module.json new file mode 100644 index 00000000..9c3279c8 --- /dev/null +++ b/modules/zfs/module.json @@ -0,0 +1,115 @@ +{ + "module": "zfs", + "version": "1", + "capabilities": [ + "service-manager" + ], + "emits": [ + "snapshot.taken" + ], + "tools": [ + "zfs_datasets", + "zfs_snapshots", + "zfs_snapshot", + "zfs_pools", + "zfs_space", + "zfs_pool_layout", + "zfs_scrub", + "zfs_io", + "zfs_arc", + "zfs_events", + "zfs_properties", + "zfs_health" + ], + "replaces": { + "zfs_datasets": [ + "zfs list" + ], + "zfs_snapshots": [ + "zfs list -t snapshot" + ], + "zfs_snapshot": [ + "zfs snapshot" + ], + "zfs_pools": [ + "zpool list", + "zpool status -x" + ], + "zfs_space": [ + "zfs list -o space", + "zpool list -v" + ], + "zfs_pool_layout": [ + "zpool status", + "zpool status -v" + ], + "zfs_scrub": [ + "zpool history -i" + ], + "zfs_io": [ + "zpool iostat", + "zpool iostat -v" + ], + "zfs_arc": [ + "arcstat", + "arc_summary", + "cat /proc/spl/kstat/zfs/arcstats" + ], + "zfs_events": [ + "zpool events", + "zpool events -v" + ], + "zfs_properties": [ + "zfs get all", + "zfs get" + ] + }, + "resources": [ + { + "id": "watch", + "type": "process", + "name": "zfs-watch", + "artifact": "tools", + "run": [ + "./zfs-tools", + "watch" + ], + "health": { + "kind": "tool", + "tool": "zfs_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "events", + "type": "process", + "name": "zfs-events", + "artifact": "tools", + "run": [ + "./zfs-tools", + "events" + ], + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/zfs-tools", + "binary": "zfs-tools", + "loads": [ + "zfs-tools" + ] + } + ] + } +}