From 5ab8bb7e62bce7418e1a4c2cef3f751c31faa4c7 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:00:42 +0100 Subject: [PATCH] Z1 Z2 Z3 zfs-manifest: packages and values, files and units, the apply and the run in the declared order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest declares zfs-utils and zfs-dkms, renames scrub-pool to pool, and declares every new value not managed until set; its config file is ${config-json}, the options file, ZED's folder, rc, functions, handlers and drop-in each under their condition, the three ZFS units, the apply (run-once, restarted by its config file) and the run before the distribution's scrub and trim timers. One commit: the three subtasks change the one file module.json. Builds: design 42 zfs-owns-zfs §6, §7, §8, §9, §11, §12, §13, §14, §15, §18, §20, A-1, A-2, A-3, A-7, A-14, A-15, A-16, A-23 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 314 ++++--- modules/zfs/cmd/zfs-tools/templates_test.go | 32 + modules/zfs/files/all-syslog.sh | 52 ++ modules/zfs/files/config.json | 1 + .../zfs/files/resilver_finish-start-scrub.sh | 20 + modules/zfs/files/scrub.conf | 6 - modules/zfs/files/zed-functions.sh | 825 ++++++++++++++++++ modules/zfs/files/zed-mesh.conf | 4 + modules/zfs/files/zed.rc | 7 + modules/zfs/files/zfs.conf | 5 + modules/zfs/module.json | 459 ++++++++-- 11 files changed, 1526 insertions(+), 199 deletions(-) create mode 100644 modules/zfs/files/all-syslog.sh create mode 100644 modules/zfs/files/config.json create mode 100644 modules/zfs/files/resilver_finish-start-scrub.sh delete mode 100644 modules/zfs/files/scrub.conf create mode 100644 modules/zfs/files/zed-functions.sh create mode 100644 modules/zfs/files/zed-mesh.conf create mode 100644 modules/zfs/files/zed.rc create mode 100644 modules/zfs/files/zfs.conf diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index bf01bec57..c8e5f7a45 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -2,13 +2,13 @@ package main import ( "encoding/json" + "fmt" "os" "path/filepath" "regexp" "sort" "strings" "testing" - "time" ) type manifest struct { @@ -84,15 +84,203 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T } } -// ADR 0240: zfs_health on the watch, which runs as the operator account; the follower judged by its -// unit. The scheduled run is not long-running and declares none. The module installs nothing, so it never -// puts ZFS on a machine. -func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { - m, raw := readManifest(t) - byID := map[string]map[string]any{} - for _, r := range m.Resources { - byID[r["id"].(string)] = r +// manifestOf is the whole manifest as a tree, for the tests that read a field the typed view does not. +func manifestOf(t *testing.T) (map[string]any, map[string]map[string]any, map[string]int) { + t.Helper() + _, raw := readManifest(t) + var tree map[string]any + if err := json.Unmarshal([]byte(raw), &tree); err != nil { + t.Fatal(err) } + byID, order := map[string]map[string]any{}, map[string]int{} + for i, r := range tree["resources"].([]any) { + res := r.(map[string]any) + byID[res["id"].(string)] = res + order[res["id"].(string)] = i + } + return tree, byID, order +} + +// conditions are the `if-config` of a resource, as written. +func conditions(r map[string]any) string { + var out []string + l, _ := r["if-config"].([]any) + for _, c := range l { + out = append(out, c.(string)) + } + return strings.Join(out, ",") +} + +// §7, §8 (A-1), §11 (A-14): the two packages are declared and the debug one is not; `pool` renames `scrub-pool`; each new value is +// not managed until set (no default, not required), of the form and trust §11 lists. +func TestTheManifestDeclaresThePackagesAndEveryValueAsSections7To11Say(t *testing.T) { + tree, byID, _ := manifestOf(t) + for id, pkg := range map[string]string{"utils": "zfs-utils", "dkms": "zfs-dkms"} { + if r := byID[id]; r == nil || r["type"] != "package" || r["package"] != pkg || r["state"] != nil { + t.Errorf("%s: %v", id, r) + } + } + for _, r := range byID { + if r["package"] == "zfs-utils-debug" { + t.Errorf("the debug package is not the module's: %v", r) + } + } + cfg := tree["config"].(map[string]any) + if _, old := cfg["scrub-pool"]; old { + t.Errorf("scrub-pool is declared; it is only in renamed-from") + } + if p := cfg["pool"].(map[string]any); p["renamed-from"] != "scrub-pool" || p["type"] != "string" || p["trusted"] != true || p["required"] != true || p["default"] != nil { + t.Errorf("pool: %v", p) + } + for k, want := range map[string]string{"scrub-cadence": "string/false", "scrub-day": "string/false", "scrub-time": "string/false"} { + if v := cfg[k].(map[string]any); v["default"] == nil || fmt.Sprint(v["type"]) != strings.Split(want, "/")[0] { + t.Errorf("%s keeps its default: %v", k, v) + } + } + new := map[string]string{"arc-max": "number/false", "arc-min": "number/false", "module-options": "list/true", "hostid": "string/true", "zed": "string/false", + "zed-handlers": "list/true", "trim-schedule": "string/false", "trim-day": "string/false", "trim-time": "string/false", "cache-devices": "list/true", + "log-devices": "list/true", "log-mirror": "boolean/true", "pool-properties": "object/true", "dataset-properties": "object/true", "snapshots": "object/true"} + for k, want := range new { + v, ok := cfg[k].(map[string]any) + if !ok { + t.Errorf("%s is not declared", k) + continue + } + if got := fmt.Sprint(v["type"], "/", v["trusted"]); got != want || v["default"] != nil || v["required"] != nil { + t.Errorf("%s: %s with default %v required %v, want %s not managed until set", k, got, v["default"], v["required"], want) + } + } + if len(cfg) != len(new)+4 { + t.Errorf("%d values declared, want %d", len(cfg), len(new)+4) + } + if d := fmt.Sprint(cfg["hostid"].(map[string]any)["description"]); !strings.Contains(d, "check of the pool's import at the next boot") { + t.Errorf("hostid's description does not say the change needs a check of the pool's import: %s", d) + } +} + +// §9 (A-15, A-23), §12, §14 (A-2, A-7), §15: the config file is `${config-json}` and not trusted; the options file, ZED's folder, rc, +// functions, handlers and drop-in each under their condition; no link and no resource or template for /etc/hostid. +func TestTheManifestsFilesAreAsSections9To15Say(t *testing.T) { + _, byID, _ := manifestOf(t) + c := byID["config"] + if c["path"] != ConfigFile || c["mode"] != "0644" || c["trusted"] != false || c["owner"] != nil || strings.TrimSpace(templateText(t, c)) != "${config-json}" { + t.Errorf("the config file: %v", c) + } + if byID["scrub-config"] != nil { + t.Errorf("scrub-config is gone: the node-engine removes the old file as a resource that left") + } + zedFiles := map[string]string{"zed-rc": "/etc/zfs/mesh-zed.d/zed.rc", "zed-functions": "/etc/zfs/mesh-zed.d/zed-functions.sh", + "zed-handler-all-syslog": "/etc/zfs/mesh-zed.d/all-syslog.sh", "zed-handler-scrub-after-resilver": "/etc/zfs/mesh-zed.d/resilver_finish-start-scrub.sh", + "zed-drop-in": "/etc/systemd/system/zfs-zed.service.d/mesh.conf"} + for id, path := range zedFiles { + r := byID[id] + want := "zed=on" + mode := "0644" + switch id { + case "zed-handler-all-syslog": + want, mode = "zed-handlers=all-syslog", "0755" + case "zed-handler-scrub-after-resilver": + want, mode = "zed-handlers=resilver_finish-start-scrub", "0755" + } + if r == nil || r["type"] != "file" || r["path"] != path || conditions(r) != want || r["mode"] != mode { + t.Errorf("%s: %v", id, r) + } + } + if d := byID["zed-dir"]; d["type"] != "directory" || d["path"] != "/etc/zfs/mesh-zed.d" || d["unknown"] != "module" || d["mode"] != "0755" || conditions(d) != "zed=on" { + t.Errorf("the folder: %v", d) + } + if o := byID["options"]; o["path"] != "/etc/modprobe.d/zfs.conf" || o["mode"] != "0644" || conditions(o) != "arc-max,arc-min,module-options" { + t.Errorf("the options file: %v", o) + } + if d := byID["run-dir"]; d["path"] != "/run/zfs-tools" || d["unknown"] != "program" || filepath.Dir(ApplyLock) != d["path"] { + t.Errorf("the lock's folder: %v", d) + } + for id, r := range byID { + if r["type"] == "symlink" || r["type"] == "link" || r["target"] != nil { + t.Errorf("%s is a link: the mesh creates none", id) + } + if r["path"] == "/etc/hostid" { + t.Errorf("%s places /etc/hostid: the apply writes it with ZFS's own writer", id) + } + if r["type"] == "file" && strings.Contains(templateText(t, r), "/etc/hostid") { + t.Errorf("%s's template names /etc/hostid", id) + } + } +} + +// §6, §13 (A-3), §18 (A-16): the run's resource precedes the resources that stop the scrub timers, with its schedule; +// each unit has the state and condition §13 lists; the trim timers are held stopped under none, weekly and monthly; the apply is +// run-once and starts again on the config file's change. +func TestTheManifestsUnitsAndTheRunAreAsSections6And13Say(t *testing.T) { + _, byID, order := manifestOf(t) + run := byID["run"] + if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || strings.Contains(fmt.Sprint(run["schedule"]), "/") { + t.Errorf("the scheduled run: %v", run) + } + if args, _ := run["run"].([]any); len(args) != 2 || args[1] != "run" { + t.Errorf("the run's command: %v", run["run"]) + } + if RunTimer != "zfs-mesh-run.timer" || RunService != "zfs-mesh-run.service" { + t.Errorf("the tools read %s and %s", RunTimer, RunService) + } + for _, id := range []string{"distribution-weekly", "distribution-monthly", "distribution-trim-weekly", "distribution-trim-monthly"} { + if r := byID[id]; r["state"] != "stopped" || r["boot"] != "disabled" || order["run"] > order[id] { + t.Errorf("%s: %v (the run is at %d, it at %d)", id, r, order["run"], order[id]) + } + } + for c, unit := range map[string]string{"distribution-weekly": "zfs-scrub-weekly@${config:pool}.timer", "distribution-monthly": "zfs-scrub-monthly@${config:pool}.timer", + "distribution-trim-weekly": "zfs-trim-weekly@${config:pool}.timer", "distribution-trim-monthly": "zfs-trim-monthly@${config:pool}.timer"} { + want := "" + if strings.Contains(c, "trim") { + want = "trim-schedule=none,trim-schedule=weekly,trim-schedule=monthly" + } + if byID[c]["unit"] != unit || conditions(byID[c]) != want { + t.Errorf("%s: %v", c, byID[c]) + } + } + units := map[string][3]string{"import-cache": {"zfs-import-cache.service", "running", "enabled"}, "mount": {"zfs-mount.service", "running", "enabled"}, + "target": {"zfs.target", "running", "enabled"}, "import-scan": {"zfs-import-scan.service", "", ""}, "share": {"zfs-share.service", "", ""}, + "volume-wait": {"zfs-volume-wait.service", "", ""}, "import-target": {"zfs-import.target", "", ""}} + for id, u := range units { + r := byID[id] + got := [3]string{fmt.Sprint(r["unit"]), fmt.Sprint(orNone(r["state"])), fmt.Sprint(orNone(r["boot"]))} + if r == nil || got != u || conditions(r) != "" { + t.Errorf("%s: %v, want %v", id, got, u) + } + } + on, off := byID["zed-on"], byID["zed-off"] + if on["unit"] != "zfs-zed.service" || on["state"] != "running" || on["boot"] != "enabled" || conditions(on) != "zed=on" || + off["unit"] != "zfs-zed.service" || off["state"] != "stopped" || off["boot"] != "disabled" || conditions(off) != "zed=off" { + t.Errorf("zed: %v %v", on, off) + } + if got := fmt.Sprint(on["restart-on"]); !strings.Contains(got, "zed-rc") || !strings.Contains(got, "zed-drop-in") || !strings.Contains(got, "zed-handler-all-syslog") { + t.Errorf("zed restarts on %v", got) + } + a := byID["apply"] + if a["type"] != "process" || a["name"] != "zfs-config-apply" || a["run-once"] != true || fmt.Sprint(a["restart-on"]) != "[config]" || a["schedule"] != nil || a["user"] != nil { + t.Errorf("the apply is run-once and starts on the config file's change: %v", a) + } + if order["config"] > order["apply"] || order["utils"] > order["config"] || order["options"] > order["apply"] || order["apply"] > order["run"] { + t.Errorf("the order: packages, config file and options before the apply, the apply before the run: %v", order) + } + for _, r := range byID { + if p, _ := r["path"].(string); strings.HasPrefix(p, "/etc/systemd/system/") && !strings.HasPrefix(p, "/etc/systemd/system/zfs-zed.service.d") { + t.Errorf("the module writes a unit file of its own: %s", p) + } + } +} + +func orNone(v any) any { + if v == nil { + return "" + } + return v +} + +// ADR 0240: zfs_health on the watch, which runs as the operator account; the follower judged by its unit. +// The scheduled run is not long-running and declares none. +func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { + _, byID, _ := manifestOf(t) w := byID["watch"] if h, _ := w["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "zfs_health" || w["user"] != "${machine:account}" { t.Errorf("the watch runs as the operator account and carries zfs_health: %v", w) @@ -101,119 +289,13 @@ func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { if h, _ := e["health"].(map[string]any); h["kind"] != "unit" || e["user"] != nil { t.Errorf("the follower runs as root, judged by its unit: %v", e) } - if d := byID["state-dir"]; d["path"] != "/var/lib/zfs-tools" || d["owner"] != "${machine:account}" || - filepath.Dir(ErrorsFile) != d["path"] { + if d := byID["state-dir"]; d["path"] != "/var/lib/zfs-tools" || d["owner"] != "${machine:account}" || filepath.Dir(ErrorsFile) != d["path"] { t.Errorf("the state the account keeps: %v", d) } // The scheduled run's state is root's alone, apart from the account's (third review). if d := byID["scrub-state-dir"]; d["path"] != filepath.Dir(ServedFile) || d["mode"] != "0700" || d["owner"] != nil { t.Errorf("the state the run keeps: %v", d) } - if strings.Contains(raw, `"package"`) { - t.Errorf("the module installs no package: ZFS is the machine's, and its kernel module is built against its kernel") - } -} - -// One schedule per pool, and no setting in any unit's name but the distribution's timers', which the -// controller fills (mesh-controller #171) and checks. The module's own schedule is the node-engine's timer -// for the scheduled run, a fixed name; the day and time reach only the configuration, which the run checks. -// The run comes before the distribution's timers are stopped, so a run that cannot be installed stops the -// apply before the timer that scrubs the pool today is touched. -func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { - m, raw := readManifest(t) - byID := map[string]map[string]any{} - order := map[string]int{} - for i, r := range m.Resources { - byID[r["id"].(string)] = r - order[r["id"].(string)] = i - } - run := byID["run"] - args, _ := run["run"].([]any) - if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || - len(args) != 2 || args[1] != "run" { - t.Errorf("the scheduled run: %v", run) - } - // The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never - // starts (novox/hq issue 1979, on the home server 2026-11-12). A list of minutes is written as a calendar - // systemd takes ("*-*-* *:0,15,30,45:00"), so the schedule names its minutes and never a step. - if s, _ := run["schedule"].(string); strings.Contains(s, "/") { - t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s) - } - if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" { - t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"]) - } - for _, c := range []string{"weekly", "monthly"} { - d := byID["distribution-"+c] - if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" { - t.Errorf("the distribution's %s timer: %v", c, d) - } - if order["run"] > order["distribution-"+c] { - t.Errorf("the distribution's %s timer is stopped before the run is installed", c) - } - } - for _, r := range m.Resources { - p, _ := r["path"].(string) - if strings.HasPrefix(p, "/etc/systemd/") { - t.Errorf("the module writes a unit file of its own: %s", p) - } - } - // The configuration's text is its template's (novox/hq ADR 0349 §2): the manifest itself names none of them. - conf := templateText(t, byID["scrub-config"]) - if strings.Contains(raw, "scrub-day}") || !strings.Contains(conf, "${config:scrub-day}") { - t.Errorf("the day reaches something other than the configuration") - } - if n := strings.Count(conf, "${config:scrub-day}") + strings.Count(conf, "${config:scrub-time}") + strings.Count(conf, "${config:scrub-cadence}"); n != 3 { - t.Errorf("the day, time and cadence are written %d times; only into the configuration", n) - } - var declared struct { - Config map[string]map[string]any `json:"config"` - } - json.Unmarshal([]byte(raw), &declared) - if declared.Config["scrub-cadence"]["default"] != "weekly" || declared.Config["scrub-day"]["default"] != "Mon" || - declared.Config["scrub-time"]["default"] != "00:00" { - t.Errorf("defaults: %v", declared.Config) - } - if p := declared.Config["scrub-pool"]; p["required"] != true || p["default"] != nil { - t.Errorf("the pool is the operator's value: required, with no default (ADR 0112, ADR 0348 §4): %v", p) - } -} - -// The home server, composed as the controller fills its settings (pool storage, cadence monthly, the -// operator's choice): the units the node-engine is sent and the configuration the run reads. -func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { - m, _ := readManifest(t) - fill := strings.NewReplacer("${config:scrub-pool}", "storage", "${config:scrub-cadence}", "monthly", - "${config:scrub-day}", "Mon", "${config:scrub-time}", "00:00") - var units []string - var conf string - for _, r := range m.Resources { - switch { - case r["type"] == "service": - units = append(units, fill.Replace(r["unit"].(string))+" "+r["state"].(string)+"/"+r["boot"].(string)) - case r["type"] == "process" && r["schedule"] != nil: - units = append(units, r["name"].(string)+".timer running/enabled (the node-engine's, every quarter of an hour)") - case r["id"] == "scrub-config": - conf = fill.Replace(templateText(t, r)) - } - } - want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)", - "zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"} - if strings.Join(units, "\n") != strings.Join(want, "\n") { - t.Errorf("units:\n%s", strings.Join(units, "\n")) - } - for _, u := range units { - t.Log(u) - } - _ = conf - cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000) - s := ScheduleOf(cfg) - if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" { - t.Fatalf("configuration: %v", refused) - } - t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04")) - if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) { - t.Errorf("next monthly slot %s, want the first Monday of November", next) - } } // Nothing but reading, a snapshot and a scrub's start, pause or stop. Every command the bundle runs goes diff --git a/modules/zfs/cmd/zfs-tools/templates_test.go b/modules/zfs/cmd/zfs-tools/templates_test.go index 7d0718b57..8b85a438c 100644 --- a/modules/zfs/cmd/zfs-tools/templates_test.go +++ b/modules/zfs/cmd/zfs-tools/templates_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -62,3 +63,34 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { t.Errorf("the build packs files/ into %d archives, not one", packs) } } + +// §9, §12, §14 (A-15), §31: the three templates. The config file is the module's values as one JSON object; the options file holds +// config-lines for the ARC (in bytes) and the options; ZED's rc holds none of the package's notification or mail lines, and the +// scrub-after-resilver variable only its handler reads. +func TestTheConfigOptionsAndZedTemplates(t *testing.T) { + _, byID, _ := manifestOf(t) + if got := strings.TrimSpace(templateText(t, byID["config"])); got != "${config-json}" { + t.Errorf("the config file: %q", got) + } + opts := templateText(t, byID["options"]) + for _, want := range []string{"${config-lines:arc-max:options zfs zfs_arc_max=:mib-bytes}", "${config-lines:arc-min:options zfs zfs_arc_min=:mib-bytes}", + "${config-lines:module-options:options zfs }"} { + if !strings.Contains(opts, want) { + t.Errorf("the options file lacks %s", want) + } + } + rc := templateText(t, byID["zed-rc"]) + for _, bad := range []string{"ZED_NOTIFY", "ZED_EMAIL", "PUSHBULLET", "SLACK", "PUSHOVER", "NTFY", "GOTIFY"} { + for _, l := range strings.Split(rc, "\n") { + if strings.Contains(l, bad) && !strings.HasPrefix(l, "#") { + t.Errorf("zed.rc holds %s: %s", bad, l) + } + } + } + if !strings.Contains(rc, "\nZED_SCRUB_AFTER_RESILVER=1\n") { + t.Errorf("zed.rc does not enable the scrub after a resilver for its handler") + } + if drop := templateText(t, byID["zed-drop-in"]); !strings.Contains(drop, "ExecStart=/usr/bin/zed -F -d /etc/zfs/mesh-zed.d") { + t.Errorf("the drop-in does not run ZED from the module's folder: %s", drop) + } +} diff --git a/modules/zfs/files/all-syslog.sh b/modules/zfs/files/all-syslog.sh new file mode 100644 index 000000000..5f601144a --- /dev/null +++ b/modules/zfs/files/all-syslog.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +# +# Log the zevent via syslog. +# + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +zed_exit_if_ignoring_this_event + +# build a string of name=value pairs for this event +msg="eid=${ZEVENT_EID} class=${ZEVENT_SUBCLASS}" + +if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then + [ -n "${ZEVENT_POOL_GUID}" ] && msg="${msg} pool_guid=${ZEVENT_POOL_GUID}" + [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}" +else + [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'" + [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}" +fi + +# log pool state if state is anything other than 'ACTIVE' +[ -n "${ZEVENT_POOL_STATE_STR}" ] && [ "$ZEVENT_POOL_STATE" -ne 0 ] && \ + msg="${msg} pool_state=${ZEVENT_POOL_STATE_STR}" + +# Log the following payload nvpairs if they are present +[ -n "${ZEVENT_VDEV_STATE_STR}" ] && msg="${msg} vdev_state=${ZEVENT_VDEV_STATE_STR}" +[ -n "${ZEVENT_CKSUM_ALGORITHM}" ] && msg="${msg} algorithm=${ZEVENT_CKSUM_ALGORITHM}" +[ -n "${ZEVENT_ZIO_SIZE}" ] && msg="${msg} size=${ZEVENT_ZIO_SIZE}" +[ -n "${ZEVENT_ZIO_OFFSET}" ] && msg="${msg} offset=${ZEVENT_ZIO_OFFSET}" +[ -n "${ZEVENT_ZIO_PRIORITY}" ] && msg="${msg} priority=${ZEVENT_ZIO_PRIORITY}" +[ -n "${ZEVENT_ZIO_ERR}" ] && msg="${msg} err=${ZEVENT_ZIO_ERR}" +[ -n "${ZEVENT_ZIO_FLAGS}" ] && msg="${msg} flags=$(printf '0x%x' "${ZEVENT_ZIO_FLAGS}")" + +# log delays that are >= 10 milisec +[ -n "${ZEVENT_ZIO_DELAY}" ] && [ "$ZEVENT_ZIO_DELAY" -gt 10000000 ] && \ + msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms" + +# list the bookmark data together +# shellcheck disable=SC2153 +[ -n "${ZEVENT_ZIO_OBJSET}" ] && \ + msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}" + +zed_log_msg "${msg}" + +exit 0 diff --git a/modules/zfs/files/config.json b/modules/zfs/files/config.json new file mode 100644 index 000000000..0984e89a2 --- /dev/null +++ b/modules/zfs/files/config.json @@ -0,0 +1 @@ +${config-json} diff --git a/modules/zfs/files/resilver_finish-start-scrub.sh b/modules/zfs/files/resilver_finish-start-scrub.sh new file mode 100644 index 000000000..cafce6fde --- /dev/null +++ b/modules/zfs/files/resilver_finish-start-scrub.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# resilver_finish-start-scrub.sh +# Run a scrub after a resilver +# +# Exit codes: +# 1: Internal error +# 2: Script wasn't enabled in zed.rc +# 3: Scrubs are automatically started for sequential resilvers +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2 +[ "${ZEVENT_RESILVER_TYPE}" != "sequential" ] || exit 3 +[ -n "${ZEVENT_POOL}" ] || exit 1 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 1 +zed_check_cmd "${ZPOOL}" || exit 1 + +zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}" +"${ZPOOL}" scrub "${ZEVENT_POOL}" diff --git a/modules/zfs/files/scrub.conf b/modules/zfs/files/scrub.conf deleted file mode 100644 index daccbd93f..000000000 --- a/modules/zfs/files/scrub.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Written by the mesh (module zfs) from this machine's assignment. Replaced on every push; change the -# settings scrub-pool, scrub-cadence, scrub-day and scrub-time, never this file. The module's tools read it. -pool=${config:scrub-pool} -cadence=${config:scrub-cadence} -day=${config:scrub-day} -time=${config:scrub-time} diff --git a/modules/zfs/files/zed-functions.sh b/modules/zfs/files/zed-functions.sh new file mode 100644 index 000000000..78d8f658d --- /dev/null +++ b/modules/zfs/files/zed-functions.sh @@ -0,0 +1,825 @@ +#!/bin/sh +# shellcheck disable=SC2154,SC3043 +# zed-functions.sh +# +# ZED helper functions for use in ZEDLETs + + +# Variable Defaults +# +: "${ZED_LOCKDIR:="/var/lock"}" +: "${ZED_NOTIFY_INTERVAL_SECS:=3600}" +: "${ZED_NOTIFY_VERBOSE:=0}" +: "${ZED_RUNDIR:="/var/run"}" +: "${ZED_SYSLOG_PRIORITY:="daemon.notice"}" +: "${ZED_SYSLOG_TAG:="zed"}" + +ZED_FLOCK_FD=8 + + +# zed_check_cmd (cmd, ...) +# +# For each argument given, search PATH for the executable command [cmd]. +# Log a message if [cmd] is not found. +# +# Arguments +# cmd: name of executable command for which to search +# +# Return +# 0 if all commands are found in PATH and are executable +# n for a count of the command executables that are not found +# +zed_check_cmd() +{ + local cmd + local rv=0 + + for cmd; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + zed_log_err "\"${cmd}\" not installed" + rv=$((rv + 1)) + fi + done + return "${rv}" +} + + +# zed_log_msg (msg, ...) +# +# Write all argument strings to the system log. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# +# Return +# nothing +# +zed_log_msg() +{ + logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "$@" +} + + +# zed_log_err (msg, ...) +# +# Write an error message to the system log. This message will contain the +# script name, EID, and all argument strings. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# ZEVENT_EID +# +# Return +# nothing +# +zed_log_err() +{ + zed_log_msg "error: ${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" +} + + +# zed_lock (lockfile, [fd]) +# +# Obtain an exclusive (write) lock on [lockfile]. If the lock cannot be +# immediately acquired, wait until it becomes available. +# +# Every zed_lock() must be paired with a corresponding zed_unlock(). +# +# By default, flock-style locks associate the lockfile with file descriptor 8. +# The bash manpage warns that file descriptors >9 should be used with care as +# they may conflict with file descriptors used internally by the shell. File +# descriptor 9 is reserved for zed_rate_limit(). If concurrent locks are held +# within the same process, they must use different file descriptors (preferably +# decrementing from 8); otherwise, obtaining a new lock with a given file +# descriptor will release the previous lock associated with that descriptor. +# +# Arguments +# lockfile: pathname of the lock file; the lock will be stored in +# ZED_LOCKDIR unless the pathname contains a "/". +# fd: integer for the file descriptor used by flock (OPTIONAL unless holding +# concurrent locks) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_lock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local umask_bak + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + umask_bak="$(umask)" + umask 077 + + # Obtain a lock on the file bound to the given file descriptor. + # + eval "exec ${fd}>> '${lockfile}'" + if ! err="$(flock --exclusive "${fd}" 2>&1)"; then + zed_log_err "failed to lock \"${lockfile}\": ${err}" + fi + + umask "${umask_bak}" +} + + +# zed_unlock (lockfile, [fd]) +# +# Release the lock on [lockfile]. +# +# Arguments +# lockfile: pathname of the lock file +# fd: integer for the file descriptor used by flock (must match the file +# descriptor passed to the zed_lock function call) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_unlock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + # Release the lock and close the file descriptor. + if ! err="$(flock --unlock "${fd}" 2>&1)"; then + zed_log_err "failed to unlock \"${lockfile}\": ${err}" + fi + eval "exec ${fd}>&-" +} + + +# zed_notify (subject, pathname) +# +# Send a notification via all available methods. +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Return +# 0: notification succeeded via at least one method +# 1: notification failed +# 2: no notification methods configured +# +zed_notify() +{ + local subject="$1" + local pathname="$2" + local num_success=0 + local num_failure=0 + + zed_notify_email "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushbullet "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_slack_webhook "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushover "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_ntfy "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_gotify "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + [ "${num_success}" -gt 0 ] && return 0 + [ "${num_failure}" -gt 0 ] && return 1 + return 2 +} + + +# zed_notify_email (subject, pathname) +# +# Send a notification via email to the address specified by ZED_EMAIL_ADDR. +# +# Requires the mail executable to be installed in the standard PATH, or +# ZED_EMAIL_PROG to be defined with the pathname of an executable capable of +# reading a message body from stdin. +# +# Command-line options to the mail executable can be specified in +# ZED_EMAIL_OPTS. This undergoes the following keyword substitutions: +# - @ADDRESS@ is replaced with the space-delimited recipient email address(es) +# - @SUBJECT@ is replaced with the notification subject +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_EMAIL_PROG +# ZED_EMAIL_OPTS +# ZED_EMAIL_ADDR +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_email() +{ + local subject="${1:-"ZED notification"}" + local pathname="${2:-"/dev/null"}" + + : "${ZED_EMAIL_PROG:="mail"}" + : "${ZED_EMAIL_OPTS:="-s '@SUBJECT@' @ADDRESS@"}" + + # For backward compatibility with ZED_EMAIL. + if [ -n "${ZED_EMAIL}" ] && [ -z "${ZED_EMAIL_ADDR}" ]; then + ZED_EMAIL_ADDR="${ZED_EMAIL}" + fi + [ -n "${ZED_EMAIL_ADDR}" ] || return 2 + + zed_check_cmd "${ZED_EMAIL_PROG}" || return 1 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err \ + "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\"" + return 1 + fi + + # construct cmdline options + ZED_EMAIL_OPTS_PARSED="$(echo "${ZED_EMAIL_OPTS}" \ + | sed -e "s/@ADDRESS@/${ZED_EMAIL_ADDR}/g" \ + -e "s/@SUBJECT@/${subject}/g")" + + # pipe message to email prog + # shellcheck disable=SC2086,SC2248 + { + # no subject passed as option? + if [ "${ZED_EMAIL_OPTS%@SUBJECT@*}" = "${ZED_EMAIL_OPTS}" ] ; then + # inject subject header + printf "Subject: %s\n" "${subject}" + # The following empty line is needed to separate the header from the + # body of the message. Otherwise programs like sendmail will skip + # everything up to the first empty line (or wont send an email at + # all) and will still exit with exit code 0 + printf "\n" + fi + # output message + cat "${pathname}" + } | + eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS_PARSED} >/dev/null 2>&1 + rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}" + return 1 + fi + return 0 +} + + +# zed_notify_pushbullet (subject, pathname) +# +# Send a notification via Pushbullet . +# The access token (ZED_PUSHBULLET_ACCESS_TOKEN) identifies this client to the +# Pushbullet server. The optional channel tag (ZED_PUSHBULLET_CHANNEL_TAG) is +# for pushing to notification feeds that can be subscribed to; if a channel is +# not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://docs.pushbullet.com/ +# https://www.pushbullet.com/security +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHBULLET_ACCESS_TOKEN +# ZED_PUSHBULLET_CHANNEL_TAG +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushbullet() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="https://api.pushbullet.com/v2/pushes" + + [ -n "${ZED_PUSHBULLET_ACCESS_TOKEN}" ] || return 2 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "pushbullet cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Push to a channel if one is configured. + # + [ -n "${ZED_PUSHBULLET_CHANNEL_TAG}" ] && msg_tag="$(printf \ + '"channel_tag": "%s", ' "${ZED_PUSHBULLET_CHANNEL_TAG}")" + + # Construct the JSON message for pushing a note. + # + msg_json="$(printf '{%s"type": "note", "title": "%s", "body": "%s"}' \ + "${msg_tag}" "${subject}" "${msg_body}")" + + # Send the POST request and check for errors. + # + msg_out="$(curl -u "${ZED_PUSHBULLET_ACCESS_TOKEN}:" -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushbullet \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_slack_webhook (subject, pathname) +# +# Notification via Slack Webhook . +# The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the +# Slack channel. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://api.slack.com/incoming-webhooks +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_SLACK_WEBHOOK_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_slack_webhook() +{ + [ -n "${ZED_SLACK_WEBHOOK_URL}" ] || return 2 + + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="${ZED_SLACK_WEBHOOK_URL}" + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "slack webhook cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Construct the JSON message for posting. + # shellcheck disable=SC2016 + # + msg_json="$(printf '{"text": "*%s*\\n```%s```"}' "${subject}" "${msg_body}" )" + + # Send the POST request and check for errors. + # + msg_out="$(curl -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "slack webhook \"${msg_err}"\" + return 1 + fi + return 0 +} + +# zed_notify_pushover (subject, pathname) +# +# Send a notification via Pushover . +# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the +# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or +# group to which the notification will be sent. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://pushover.net/api +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHOVER_TOKEN +# ZED_PUSHOVER_USER +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushover() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + local url="https://api.pushover.net/1/messages.json" + + [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2 + + if [ ! -r "${pathname}" ]; then + zed_log_err "pushover cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + msg_out="$( \ + curl \ + --form-string "token=${ZED_PUSHOVER_TOKEN}" \ + --form-string "user=${ZED_PUSHOVER_USER}" \ + --form-string "message=${msg_body}" \ + --form-string "title=${subject}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushover \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_ntfy (subject, pathname) +# +# Send a notification via Ntfy.sh . +# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification +# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the +# self-hosted or provided hosted ntfy service location. The ntfy access token +# (ZED_NTFY_ACCESS_TOKEN) reprsents an +# access token that could be used if a topic is read/write protected. If a +# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://docs.ntfy.sh +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_NTFY_TOPIC +# ZED_NTFY_ACCESS_TOKEN (OPTIONAL) +# ZED_NTFY_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_ntfy() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_NTFY_TOPIC}" ] || return 2 + local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "ntfy cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then + msg_out="$( \ + curl \ + -u ":${ZED_NTFY_ACCESS_TOKEN}" \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "ntfy \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_gotify (subject, pathname) +# +# Send a notification via Gotify . +# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location. +# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a +# Gotify application token which is associated with a message. +# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the +# default or configured priority at the Gotify server for the application. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://gotify.net/docs/index +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_GOTIFY_URL +# ZED_GOTIFY_APPTOKEN +# ZED_GOTIFY_PRIORITY +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_gotify() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2 + local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "gotify cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + --form-string "priority=${ZED_GOTIFY_PRIORITY}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "gotify \"${msg_err}"\" + return 1 + fi + return 0 +} + + + +# zed_rate_limit (tag, [interval]) +# +# Check whether an event of a given type [tag] has already occurred within the +# last [interval] seconds. +# +# This function obtains a lock on the statefile using file descriptor 9. +# +# Arguments +# tag: arbitrary string for grouping related events to rate-limit +# interval: time interval in seconds (OPTIONAL) +# +# Globals +# ZED_NOTIFY_INTERVAL_SECS +# ZED_RUNDIR +# +# Return +# 0 if the event should be processed +# 1 if the event should be dropped +# +# State File Format +# time;tag +# +zed_rate_limit() +{ + local tag="$1" + local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}" + local lockfile="zed.zedlet.state.lock" + local lockfile_fd=9 + local statefile="${ZED_RUNDIR}/zed.zedlet.state" + local time_now + local time_prev + local umask_bak + local rv=0 + + [ -n "${tag}" ] || return 0 + + zed_lock "${lockfile}" "${lockfile_fd}" + time_now="$(date +%s)" + time_prev="$(grep -E "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + | tail -1 | cut -d\; -f1)" + + if [ -n "${time_prev}" ] \ + && [ "$((time_now - time_prev))" -lt "${interval}" ]; then + rv=1 + else + umask_bak="$(umask)" + umask 077 + grep -E -v "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + > "${statefile}.$$" + echo "${time_now};${tag}" >> "${statefile}.$$" + mv -f "${statefile}.$$" "${statefile}" + umask "${umask_bak}" + fi + + zed_unlock "${lockfile}" "${lockfile_fd}" + return "${rv}" +} + + +# zed_guid_to_pool (guid) +# +# Convert a pool GUID into its pool name (like "tank") +# Arguments +# guid: pool GUID (decimal or hex) +# +# Return +# Pool name +# +zed_guid_to_pool() +{ + if [ -z "$1" ] ; then + return + fi + + guid="$(printf "%u" "$1")" + $ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}' +} + +# zed_exit_if_ignoring_this_event +# +# Exit the script if we should ignore this event, as determined by +# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc. +# This function assumes you've imported the normal zed variables. +zed_exit_if_ignoring_this_event() +{ + if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_INCLUDE});; + *) exit 0;; + esac" + elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;; + *);; + esac" + fi +} diff --git a/modules/zfs/files/zed-mesh.conf b/modules/zfs/files/zed-mesh.conf new file mode 100644 index 000000000..196be4669 --- /dev/null +++ b/modules/zfs/files/zed-mesh.conf @@ -0,0 +1,4 @@ +# Written by the mesh (module zfs). ZED runs from the module's own folder, which holds no link. +[Service] +ExecStart= +ExecStart=/usr/bin/zed -F -d /etc/zfs/mesh-zed.d diff --git a/modules/zfs/files/zed.rc b/modules/zfs/files/zed.rc new file mode 100644 index 000000000..562b9f594 --- /dev/null +++ b/modules/zfs/files/zed.rc @@ -0,0 +1,7 @@ +# Written by the mesh (module zfs). Replaced on every push; change the settings zed and zed-handlers, never this file. +# It holds the package's own variables but its notification and mail lines: a pool's fault reaches the operator once, +# from the event follower's finding, never from a ZED handler. +ZED_USE_ENCLOSURE_LEDS=1 +ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event" +# Read only by the scrub-after-resilver handler, which is placed only while zed-handlers names it. +ZED_SCRUB_AFTER_RESILVER=1 diff --git a/modules/zfs/files/zfs.conf b/modules/zfs/files/zfs.conf new file mode 100644 index 000000000..edc788e72 --- /dev/null +++ b/modules/zfs/files/zfs.conf @@ -0,0 +1,5 @@ +# Written by the mesh (module zfs) from this machine's assignment. Replaced on every push; change the +# settings arc-max, arc-min and module-options, never this file. ZFS reads it when its kernel module loads. +${config-lines:arc-max:options zfs zfs_arc_max=:mib-bytes} +${config-lines:arc-min:options zfs zfs_arc_min=:mib-bytes} +${config-lines:module-options:options zfs } diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 200e26364..2f119f5ab 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -2,52 +2,151 @@ "module": "zfs", "version": "1", "capabilities": [ + "package-manager", "service-manager" ], "emits": [ "snapshot.taken", "scrub.requested" ], - "config": { - "scrub-pool": { - "type": "string", - "description": "The name of the ZFS pool the module scrubs; the operator's, with no default.", - "trusted": true, - "required": true - }, - "scrub-cadence": { - "type": "string", - "description": "How often the pool named by scrub-pool is scrubbed: weekly, or monthly on the first scrub-day of each month. Weekly is what the distribution's timer did on the home server before the module, so taking it over moves nothing until the operator chooses.", - "enum": [ - "weekly", - "monthly" - ], - "trusted": true, - "default": "weekly" - }, - "scrub-day": { - "type": "string", - "description": "The weekday the scrub starts, Mon to Sun; for a monthly scrub, the first such day of the month. Checked by the scheduled run, which refuses any other value and runs nothing; Monday is when the distribution's weekly timer fired.", - "enum": [ - "Mon", - "Tue", - "Wed", - "Thu", - "Fri", - "Sat", - "Sun" - ], - "trusted": true, - "default": "Mon" - }, - "scrub-time": { - "type": "string", - "description": "The time the scrub starts, HH:MM from 00:00 to 23:59 in the machine's time zone; the scheduled run checks it every quarter of an hour, so the scrub starts within 15 minutes of it. Midnight is when the distribution's weekly timer fired.", - "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", - "trusted": true, - "default": "00:00" - } + "config": { + "pool": { + "type": "string", + "description": "The name of the ZFS pool the module owns: scrubs, trims, takes scheduled ZFS snapshots of and converges; the operator's, with no default.", + "trusted": true, + "required": true, + "renamed-from": "scrub-pool" }, + "scrub-cadence": { + "type": "string", + "description": "How often the pool named by pool is scrubbed: weekly, or monthly on the first scrub-day of each month. Weekly is what the distribution's timer did on the home server before the module, so taking it over moves nothing until the operator chooses.", + "enum": [ + "weekly", + "monthly" + ], + "trusted": true, + "default": "weekly" + }, + "scrub-day": { + "type": "string", + "description": "The weekday the scrub starts, Mon to Sun; for a monthly scrub, the first such day of the month. Checked by the scheduled run, which refuses any other value and runs nothing; Monday is when the distribution's weekly timer fired.", + "enum": [ + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", + "Sun" + ], + "trusted": true, + "default": "Mon" + }, + "scrub-time": { + "type": "string", + "description": "The time the scrub starts, HH:MM from 00:00 to 23:59 in the machine's time zone; the scheduled run checks it every quarter of an hour, so the scrub starts within 15 minutes of it. Midnight is when the distribution's weekly timer fired.", + "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", + "trusted": true, + "default": "00:00" + }, + "arc-max": { + "type": "number", + "description": "The most ZFS's read cache in memory (the ARC) may hold, in whole MiB: at least 64 and below the node's memory. Written live and into the options file. Not managed until set.", + "minimum": 64, + "trusted": false + }, + "arc-min": { + "type": "number", + "description": "The least the ARC keeps, in whole MiB, not above arc-max. Written live and into the options file. Not managed until set.", + "minimum": 0, + "trusted": false + }, + "module-options": { + "type": "list", + "description": "Kernel module parameters, each name=digits, from l2arc_write_max, l2arc_write_boost, l2arc_headroom, l2arc_noprefetch, l2arc_rebuild_enabled, zfs_txg_timeout, zfs_dirty_data_max, zfs_scan_vdev_limit, zfs_scrub_min_time_ms, zfs_resilver_min_time_ms and zfs_arc_sys_free (arc-max and arc-min hold the ARC's). Written into the options file; a live one is also written live, the others take effect at the next load of the kernel module. An empty list is not managed.", + "trusted": true + }, + "hostid": { + "type": "string", + "description": "The node's ZFS id: eight hexadecimal digits, not all zero, written to /etc/hostid with ZFS's own writer only when the pool's label holds 0 or this value. Changing it needs a check of the pool's import at the next boot; the module never reboots. Not managed until set.", + "pattern": "^[0-9a-fA-F]{8}$", + "trusted": true + }, + "zed": { + "type": "string", + "description": "Whether the ZFS event daemon runs from the module's own folder: on or off. Not managed until set.", + "enum": [ + "on", + "off" + ], + "trusted": false + }, + "zed-handlers": { + "type": "list", + "description": "The event handlers placed in the module's folder: all-syslog (the journal handler) and resilver_finish-start-scrub (a scrub after a resilver). Empty is not managed.", + "trusted": true + }, + "trim-schedule": { + "type": "string", + "description": "How often the pool is trimmed by the scheduled run: none, weekly, or monthly on the first trim-day of each month. Any set value holds the distribution's trim timers stopped. Needs trim-day and trim-time for weekly and monthly. Not managed until set.", + "enum": [ + "none", + "weekly", + "monthly" + ], + "trusted": false + }, + "trim-day": { + "type": "string", + "description": "The weekday the trim starts, Mon to Sun; for monthly, the first such day of the month. Not managed until set.", + "enum": [ + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", + "Sun" + ], + "trusted": false + }, + "trim-time": { + "type": "string", + "description": "The time the trim starts, HH:MM in the machine's time zone, within 15 minutes. Not managed until set.", + "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", + "trusted": false + }, + "cache-devices": { + "type": "list", + "description": "The pool's cache devices, by their names under /dev/disk/by-id: letters, digits, -, _, . and : only. A kernel-named member is re-added by its stable name; an empty list removes nothing and is not managed.", + "trusted": true + }, + "log-devices": { + "type": "list", + "description": "The pool's log devices, by their names under /dev/disk/by-id. An empty list removes nothing and is not managed.", + "trusted": true + }, + "log-mirror": { + "type": "boolean", + "description": "Whether log-devices are one mirror: true needs exactly two names, added as one mirror; otherwise each is a log device of its own. Not managed until set.", + "trusted": true + }, + "pool-properties": { + "type": "object", + "description": "The pool's properties: autotrim, autoexpand, autoreplace (on, off or default) and failmode (wait, continue, panic or default). Not managed until set.", + "trusted": true + }, + "dataset-properties": { + "type": "object", + "description": "Per dataset of the pool, its properties: compression, atime, relatime, recordsize, primarycache, secondarycache, sync, logbias, quota and reservation, each in its own form; never sharenfs or sharesmb. Not managed until set.", + "trusted": true + }, + "snapshots": { + "type": "object", + "description": "The scheduled ZFS snapshots of every filesystem of the pool: {\"none\": true}, or keep counts hourly, daily, weekly and monthly, each a whole number from 0 to 168; a count of 0 takes none of that kind and removes none. Only snapshots the module took, named mesh-- and recorded in its ledger, are ever removed. Not managed until set.", + "trusted": true + } + }, "tools": [ "zfs_datasets", "zfs_snapshots", @@ -129,6 +228,224 @@ ] }, "resources": [ + { + "id": "utils", + "type": "package", + "package": "zfs-utils" + }, + { + "id": "dkms", + "type": "package", + "package": "zfs-dkms" + }, + { + "id": "state-dir", + "type": "directory", + "path": "/var/lib/zfs-tools", + "mode": "0755", + "owner": "${machine:account}", + "unknown": "program" + }, + { + "id": "cursor-dir", + "type": "directory", + "path": "/run/zfs-events", + "mode": "0700", + "unknown": "program" + }, + { + "id": "run-dir", + "type": "directory", + "path": "/run/zfs-tools", + "mode": "0755", + "unknown": "program" + }, + { + "id": "config-dir", + "type": "directory", + "path": "/etc/zfs-tools", + "mode": "0755", + "unknown": "program" + }, + { + "id": "scrub-state-dir", + "type": "directory", + "path": "/var/lib/zfs-scrub", + "mode": "0700", + "unknown": "program" + }, + { + "id": "config", + "type": "file", + "path": "/etc/zfs-tools/config.json", + "mode": "0644", + "template": "files/config.json", + "trusted": false + }, + { + "id": "options", + "type": "file", + "path": "/etc/modprobe.d/zfs.conf", + "mode": "0644", + "template": "files/zfs.conf", + "if-config": [ + "arc-max", + "arc-min", + "module-options" + ] + }, + { + "id": "zed-dir", + "type": "directory", + "path": "/etc/zfs/mesh-zed.d", + "mode": "0755", + "unknown": "module", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-rc", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/zed.rc", + "mode": "0644", + "template": "files/zed.rc", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-functions", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/zed-functions.sh", + "mode": "0644", + "template": "files/zed-functions.sh", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-handler-all-syslog", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/all-syslog.sh", + "mode": "0755", + "template": "files/all-syslog.sh", + "if-config": [ + "zed-handlers=all-syslog" + ] + }, + { + "id": "zed-handler-scrub-after-resilver", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/resilver_finish-start-scrub.sh", + "mode": "0755", + "template": "files/resilver_finish-start-scrub.sh", + "if-config": [ + "zed-handlers=resilver_finish-start-scrub" + ] + }, + { + "id": "zed-drop-ins", + "type": "directory", + "path": "/etc/systemd/system/zfs-zed.service.d", + "mode": "0755", + "unknown": "program", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-drop-in", + "type": "file", + "path": "/etc/systemd/system/zfs-zed.service.d/mesh.conf", + "mode": "0644", + "template": "files/zed-mesh.conf", + "if-config": [ + "zed=on" + ] + }, + { + "id": "import-cache", + "type": "service", + "unit": "zfs-import-cache.service", + "state": "running", + "boot": "enabled" + }, + { + "id": "mount", + "type": "service", + "unit": "zfs-mount.service", + "state": "running", + "boot": "enabled" + }, + { + "id": "target", + "type": "service", + "unit": "zfs.target", + "state": "running", + "boot": "enabled" + }, + { + "id": "import-scan", + "type": "service", + "unit": "zfs-import-scan.service" + }, + { + "id": "share", + "type": "service", + "unit": "zfs-share.service" + }, + { + "id": "volume-wait", + "type": "service", + "unit": "zfs-volume-wait.service" + }, + { + "id": "import-target", + "type": "service", + "unit": "zfs-import.target" + }, + { + "id": "zed-on", + "type": "service", + "unit": "zfs-zed.service", + "state": "running", + "boot": "enabled", + "if-config": [ + "zed=on" + ], + "restart-on": [ + "zed-rc", + "zed-functions", + "zed-handler-all-syslog", + "zed-handler-scrub-after-resilver", + "zed-drop-in" + ] + }, + { + "id": "zed-off", + "type": "service", + "unit": "zfs-zed.service", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "zed=off" + ] + }, + { + "id": "apply", + "type": "process", + "name": "zfs-config-apply", + "artifact": "tools", + "run": [ + "./zfs-tools", + "apply" + ], + "run-once": true, + "restart-on": [ + "config" + ] + }, { "id": "watch", "type": "process", @@ -148,21 +465,6 @@ "grace": "60s" } }, - { - "id": "state-dir", - "type": "directory", - "path": "/var/lib/zfs-tools", - "mode": "0755", - "owner": "${machine:account}", - "unknown": "program" - }, - { - "id": "cursor-dir", - "type": "directory", - "path": "/run/zfs-events", - "mode": "0700", - "unknown": "program" - }, { "id": "events", "type": "process", @@ -176,27 +478,6 @@ "kind": "unit" } }, - { - "id": "config-dir", - "type": "directory", - "path": "/etc/zfs-tools", - "mode": "0755", - "unknown": "program" - }, - { - "id": "scrub-config", - "type": "file", - "path": "/etc/zfs-tools/scrub.conf", - "mode": "0644", - "template": "files/scrub.conf" - }, - { - "id": "scrub-state-dir", - "type": "directory", - "path": "/var/lib/zfs-scrub", - "mode": "0700", - "unknown": "program" - }, { "id": "run", "type": "process", @@ -211,16 +492,40 @@ { "id": "distribution-weekly", "type": "service", - "unit": "zfs-scrub-weekly@${config:scrub-pool}.timer", + "unit": "zfs-scrub-weekly@${config:pool}.timer", "state": "stopped", "boot": "disabled" }, { "id": "distribution-monthly", "type": "service", - "unit": "zfs-scrub-monthly@${config:scrub-pool}.timer", + "unit": "zfs-scrub-monthly@${config:pool}.timer", "state": "stopped", "boot": "disabled" + }, + { + "id": "distribution-trim-weekly", + "type": "service", + "unit": "zfs-trim-weekly@${config:pool}.timer", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "trim-schedule=none", + "trim-schedule=weekly", + "trim-schedule=monthly" + ] + }, + { + "id": "distribution-trim-monthly", + "type": "service", + "unit": "zfs-trim-monthly@${config:pool}.timer", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "trim-schedule=none", + "trim-schedule=weekly", + "trim-schedule=monthly" + ] } ], "build": {