From 2b699d4046b3c1da67332d02f584eb93a5b0de01 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:23:56 +0100 Subject: [PATCH 01/35] Z0 pacman-config-first: the pacman module declares its config file before its package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The apply stops at the first failure and the package fails while the config file is missing (A-26). Builds: design 42 zfs-owns-zfs §1, A-26 --- modules/pacman/cmd/pacman-tools/manifest_test.go | 12 ++++++++++++ modules/pacman/module.json | 10 +++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index 6cf1df713..45cd3f68e 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -133,3 +133,15 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { t.Fatalf("a package's file and timer come after the package: %v", order) } } + +// TestThePackageManagersConfigIsDeclaredBeforeItsPackage holds design 42 zfs-owns-zfs §1 and A-26: the +// apply stops at the first failure, and the package fails while its config file is missing. +func TestThePackageManagersConfigIsDeclaredBeforeItsPackage(t *testing.T) { + order := map[string]int{} + for i, r := range manifest(t).Resources { + order[r["id"].(string)] = i + } + if order["config"] >= order["package"] { + t.Fatalf("the config file comes before the package: %v", order) + } +} diff --git a/modules/pacman/module.json b/modules/pacman/module.json index 307c79a4e..df46ec650 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -89,11 +89,6 @@ "pacman_config" ], "resources": [ - { - "id": "package", - "type": "package", - "package": "pacman" - }, { "id": "config", "type": "file", @@ -101,6 +96,11 @@ "mode": "0644", "template": "files/pacman.conf" }, + { + "id": "package", + "type": "package", + "package": "pacman" + }, { "id": "contrib", "type": "package", From 3cb2b83a15a06dab0d4215dbdb9a70e8660ac11a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:25:48 +0100 Subject: [PATCH 02/35] Z5 zfs-config-parse: the config file's values each read in their fixed form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A value of another form is refused before anything is written or run; absent and empty are not managed. Builds: design 42 zfs-owns-zfs §9, §11, A-15, A-23 --- modules/zfs/cmd/zfs-tools/config.go | 467 +++++++++++++++++++++++ modules/zfs/cmd/zfs-tools/config_test.go | 96 +++++ 2 files changed, 563 insertions(+) create mode 100644 modules/zfs/cmd/zfs-tools/config.go create mode 100644 modules/zfs/cmd/zfs-tools/config_test.go diff --git a/modules/zfs/cmd/zfs-tools/config.go b/modules/zfs/cmd/zfs-tools/config.go new file mode 100644 index 000000000..b608a09cd --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/config.go @@ -0,0 +1,467 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +// The module's config file and the forms of its values (design 42 zfs-owns-zfs §9, §11). +// +// The node-engine writes ConfigFile as one JSON object of the values that are set (`${config-json}`, A-15); +// an unset value is absent. **A value that is absent, an empty string, an empty list or an empty object is +// not managed**: the module writes nothing, converges nothing and reports no difference for it, so a blank +// value never removes anything. A value of another form is refused here, before any file is written or +// command run; the caller says each refusal as `config.refused` and acts on none of the file (§25). + +// ConfigFile is the module's config file; the apply and the scheduled run read it at each start. +const ConfigFile = "/etc/zfs-tools/config.json" + +// Refusal is one value the parse refused: `config.refused` {key, value, reason}. +type Refusal struct { + Key string `json:"key"` + Value string `json:"value"` + Reason string `json:"reason"` +} + +// ModuleOption is one `name=digits` entry of module-options. +type ModuleOption struct{ Name, Value string } + +// Keeps are the keep counts of `snapshots`; None is the value `none`. +type Keeps struct { + None bool + Hourly, Daily, Weekly, Monthly int +} + +// Config is the module's values as set. A pointer, an empty string, a nil list or map is not managed. +type Config struct { + Pool string + ScrubCadence, ScrubDay, ScrubTime string + ArcMax, ArcMin *uint64 // MiB + ModuleOptions []ModuleOption + HostID string // eight lower-case hexadecimal digits + Zed string + ZedHandlers []string + TrimSchedule, TrimDay, TrimTime string + CacheDevices, LogDevices []string + LogMirror *bool + PoolProperties map[string]string + DatasetProperties map[string]map[string]string + Snapshots *Keeps +} + +var ( + moduleOptionNames = map[string]bool{"l2arc_write_max": true, "l2arc_write_boost": true, "l2arc_headroom": true, + "l2arc_noprefetch": true, "l2arc_rebuild_enabled": true, "zfs_txg_timeout": true, "zfs_dirty_data_max": true, + "zfs_scan_vdev_limit": true, "zfs_scrub_min_time_ms": true, "zfs_resilver_min_time_ms": true, "zfs_arc_sys_free": true} + hostIDForm = regexp.MustCompile(`^[0-9a-fA-F]{8}$`) + digitsForm = regexp.MustCompile(`^[0-9]+$`) + deviceForm = regexp.MustCompile(`^[A-Za-z0-9_.:-]+$`) + datasetForm = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.:-]*(/[A-Za-z0-9_.:-]+)*$`) + recordsizeRe = regexp.MustCompile(`^(512|1K|2K|4K|8K|16K|32K|64K|128K|256K|512K|1M|2M|4M|8M|16M)$`) + sizeForm = regexp.MustCompile(`^[0-9]+[KMGT]$`) + zedHandlerOK = map[string]bool{"all-syslog": true, "resilver_finish-start-scrub": true} +) + +// keepKinds are the kinds of `snapshots`, in the order they are said. +var keepKinds = []string{"hourly", "daily", "weekly", "monthly"} + +// MaxKeep is the bound of a keep count: a week of hours (§11), so the ledger's cap stays bounded. +const MaxKeep = 168 + +// poolPropertyForm and datasetPropertyForm hold each listed property's form (§11, §16); "" is the form's +// failure, and every property also takes `default`. +func poolPropertyForm(p, v string) string { + switch p { + case "autotrim", "autoexpand", "autoreplace": + return oneOf(v, "on", "off", "default") + case "failmode": + return oneOf(v, "wait", "continue", "panic", "default") + } + return "is not a pool property the module sets (autotrim, autoexpand, autoreplace, failmode)" +} + +func oneOf(v string, ok ...string) string { + for _, o := range ok { + if v == o { + return "" + } + } + return "is not one of " + strings.Join(ok, ", ") +} + +func datasetPropertyForm(p, v string) string { + if v == "default" { + switch p { + case "compression", "atime", "relatime", "recordsize", "primarycache", "secondarycache", "sync", "logbias", "quota", "reservation": + return "" + } + } + switch p { + case "compression": + if v == "off" || v == "lz4" || v == "zle" || v == "zstd" { + return "" + } + if n, ok := strings.CutPrefix(v, "zstd-"); ok { + if i, err := strconv.Atoi(n); err == nil && i >= 1 && i <= 19 && strconv.Itoa(i) == n { + return "" + } + } + if n, ok := strings.CutPrefix(v, "gzip-"); ok { + if i, err := strconv.Atoi(n); err == nil && i >= 1 && i <= 9 && strconv.Itoa(i) == n { + return "" + } + } + return "is not a compression of the list (off, lz4, zstd, zstd-1 to zstd-19, gzip-1 to gzip-9, zle)" + case "atime", "relatime": + return oneOf(v, "on", "off") + case "recordsize": + if !recordsizeRe.MatchString(v) { + return "is not a power of two from 512 to 16M, with K or M" + } + return "" + case "primarycache", "secondarycache": + return oneOf(v, "all", "metadata", "none") + case "sync": + return oneOf(v, "standard", "always", "disabled") + case "logbias": + return oneOf(v, "latency", "throughput") + case "quota", "reservation": + if v == "none" || sizeForm.MatchString(v) { + return "" + } + return "is not none, or digits with K, M, G or T" + } + return "is not a dataset property of the list (compression, atime, relatime, recordsize, primarycache, secondarycache, sync, logbias, quota, reservation)" +} + +type parser struct { + raw map[string]json.RawMessage + out []Refusal +} + +func (p *parser) refuse(key string, v json.RawMessage, why string) { + val := strings.TrimSpace(string(v)) + if len(val) > 200 { + val = val[:200] + "..." + } + p.out = append(p.out, Refusal{Key: key, Value: val, Reason: why}) +} + +// str reads a string value; ok is false when it is not managed (absent or empty) or refused. +func (p *parser) str(key string, form func(string) string) (string, bool) { + v, has := p.raw[key] + if !has || string(bytes.TrimSpace(v)) == "null" { + return "", false + } + var s string + if err := json.Unmarshal(v, &s); err != nil { + p.refuse(key, v, "is not a string") + return "", false + } + if s == "" { + return "", false + } + if why := form(s); why != "" { + p.refuse(key, v, why) + return "", false + } + return s, true +} + +func (p *parser) list(key string) ([]string, json.RawMessage, bool) { + v, has := p.raw[key] + if !has || string(bytes.TrimSpace(v)) == "null" { + return nil, nil, false + } + var l []string + if err := json.Unmarshal(v, &l); err != nil { + p.refuse(key, v, "is not a list of strings") + return nil, nil, false + } + if len(l) == 0 { + return nil, nil, false + } + return l, v, true +} + +func (p *parser) number(key string) (uint64, json.RawMessage, bool) { + v, has := p.raw[key] + if !has || string(bytes.TrimSpace(v)) == "null" { + return 0, nil, false + } + if t := bytes.TrimSpace(v); len(t) == 0 || t[0] == '"' { + p.refuse(key, v, "is not a whole number of MiB") + return 0, nil, false + } + var n json.Number + dec := json.NewDecoder(bytes.NewReader(v)) + dec.UseNumber() + if err := dec.Decode(&n); err != nil { + p.refuse(key, v, "is not a whole number of MiB") + return 0, nil, false + } + u, err := strconv.ParseUint(n.String(), 10, 64) + if err != nil { + p.refuse(key, v, "is not a whole number of MiB") + return 0, nil, false + } + return u, v, true +} + +func (p *parser) object(key string) (map[string]json.RawMessage, json.RawMessage, bool) { + v, has := p.raw[key] + if !has || string(bytes.TrimSpace(v)) == "null" { + return nil, nil, false + } + var o map[string]json.RawMessage + if err := json.Unmarshal(v, &o); err != nil { + p.refuse(key, v, "is not an object") + return nil, nil, false + } + if len(o) == 0 { + return nil, nil, false + } + return o, v, true +} + +func sortedKeys[V any](m map[string]V) []string { + k := make([]string, 0, len(m)) + for x := range m { + k = append(k, x) + } + sort.Strings(k) + return k +} + +// ParseConfig reads the config file's text. memMiB is the node's memory from /proc/meminfo, in MiB (0 when +// it could not be read, which refuses arc-max). It answers every refusal; a Config with any refusal is +// never acted on. +func ParseConfig(text []byte, memMiB uint64) (*Config, []Refusal) { + p := &parser{} + if err := json.Unmarshal(text, &p.raw); err != nil { + return nil, []Refusal{{Key: ConfigFile, Value: "", Reason: "is not a JSON object: " + err.Error()}} + } + c := &Config{} + c.Pool, _ = p.str("pool", func(s string) string { + if !poolName.MatchString(s) { + return "is not a pool's name" + } + return "" + }) + c.ScrubCadence, _ = p.str("scrub-cadence", func(s string) string { return oneOf(s, "weekly", "monthly") }) + c.ScrubDay, _ = p.str("scrub-day", func(s string) string { + return oneOf(s, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") + }) + c.ScrubTime, _ = p.str("scrub-time", clockForm) + c.TrimSchedule, _ = p.str("trim-schedule", func(s string) string { return oneOf(s, "none", "weekly", "monthly") }) + c.TrimDay, _ = p.str("trim-day", func(s string) string { + return oneOf(s, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") + }) + c.TrimTime, _ = p.str("trim-time", clockForm) + c.Zed, _ = p.str("zed", func(s string) string { return oneOf(s, "on", "off") }) + if h, ok := p.str("hostid", func(s string) string { + switch { + case !hostIDForm.MatchString(s): + return "is not eight hexadecimal digits" + case strings.Trim(s, "0") == "": + return "is all zero" + } + return "" + }); ok { + c.HostID = strings.ToLower(h) + } + + var arcMin json.RawMessage + if n, raw, ok := p.number("arc-max"); ok { + switch { + case n < 64: + p.refuse("arc-max", raw, "is below 64 MiB") + case memMiB == 0: + p.refuse("arc-max", raw, "cannot be checked against the node's memory, which /proc/meminfo did not say") + case n >= memMiB: + p.refuse("arc-max", raw, fmt.Sprintf("is not below the node's memory of %d MiB", memMiB)) + default: + c.ArcMax = &n + } + } + if n, raw, ok := p.number("arc-min"); ok { + arcMin = raw + c.ArcMin = &n + } + if c.ArcMin != nil && c.ArcMax != nil && *c.ArcMin > *c.ArcMax { + p.refuse("arc-min", arcMin, "is above arc-max") + c.ArcMin = nil + } + + if l, raw, ok := p.list("module-options"); ok { + seen := map[string]bool{} + for _, e := range l { + name, val, has := strings.Cut(e, "=") + switch { + case !has || !digitsForm.MatchString(val) || len(val) > 20: + p.refuse("module-options", raw, fmt.Sprintf("%q is not name=digits", e)) + case name == "zfs_arc_max" || name == "zfs_arc_min": + p.refuse("module-options", raw, name+" is held by arc-max and arc-min") + case !moduleOptionNames[name]: + p.refuse("module-options", raw, fmt.Sprintf("%q is not a parameter of the module's list", name)) + case seen[name]: + p.refuse("module-options", raw, name+" is given twice") + default: + seen[name] = true + c.ModuleOptions = append(c.ModuleOptions, ModuleOption{name, val}) + } + } + } + if l, raw, ok := p.list("zed-handlers"); ok { + seen := map[string]bool{} + for _, h := range l { + switch { + case !zedHandlerOK[h]: + p.refuse("zed-handlers", raw, fmt.Sprintf("%q is not all-syslog or resilver_finish-start-scrub", h)) + case seen[h]: + p.refuse("zed-handlers", raw, h+" is given twice") + default: + seen[h] = true + c.ZedHandlers = append(c.ZedHandlers, h) + } + } + } + devices := func(key string) []string { + l, raw, ok := p.list(key) + if !ok { + return nil + } + var good []string + for _, d := range l { + if !deviceForm.MatchString(d) || d == "." || d == ".." { + p.refuse(key, raw, fmt.Sprintf("%q is not a name under /dev/disk/by-id (letters, digits, - _ . : only)", d)) + continue + } + good = append(good, d) + } + return good + } + c.CacheDevices, c.LogDevices = devices("cache-devices"), devices("log-devices") + if v, has := p.raw["log-mirror"]; has && string(bytes.TrimSpace(v)) != "null" { + var b bool + if err := json.Unmarshal(v, &b); err != nil { + p.refuse("log-mirror", v, "is not true or false") + } else { + c.LogMirror = &b + if b && len(c.LogDevices) != 2 { + p.refuse("log-mirror", v, "is true, and log-devices does not hold exactly two names") + } + } + } + + if o, raw, ok := p.object("pool-properties"); ok { + c.PoolProperties = map[string]string{} + for _, k := range sortedKeys(o) { + var s string + if json.Unmarshal(o[k], &s) != nil { + p.refuse("pool-properties", raw, fmt.Sprintf("%s is not a string", k)) + } else if why := poolPropertyForm(k, s); why != "" { + p.refuse("pool-properties", raw, fmt.Sprintf("%s=%s %s", k, s, why)) + } else { + c.PoolProperties[k] = s + } + } + } + if o, raw, ok := p.object("dataset-properties"); ok { + c.DatasetProperties = map[string]map[string]string{} + for _, ds := range sortedKeys(o) { + var props map[string]string + switch { + case c.Pool == "": + p.refuse("dataset-properties", raw, "needs the pool set") + case ds != c.Pool && !strings.HasPrefix(ds, c.Pool+"/"): + p.refuse("dataset-properties", raw, fmt.Sprintf("%q is not the pool's name or one beginning %s/", ds, c.Pool)) + case !datasetForm.MatchString(ds): + p.refuse("dataset-properties", raw, fmt.Sprintf("%q is not a dataset's name", ds)) + case json.Unmarshal(o[ds], &props) != nil: + p.refuse("dataset-properties", raw, fmt.Sprintf("%s is not an object of strings", ds)) + default: + c.DatasetProperties[ds] = map[string]string{} + for _, k := range sortedKeys(props) { + if why := datasetPropertyForm(k, props[k]); why != "" { + p.refuse("dataset-properties", raw, fmt.Sprintf("%s %s=%s %s", ds, k, props[k], why)) + } else { + c.DatasetProperties[ds][k] = props[k] + } + } + } + } + } + if o, raw, ok := p.object("snapshots"); ok { + k := &Keeps{} + if _, none := o["none"]; none { + if len(o) != 1 { + p.refuse("snapshots", raw, "none is given with keep counts") + } + k.None = true + } else { + for _, name := range sortedKeys(o) { + var n int + if !contains(keepKinds, name) { + p.refuse("snapshots", raw, fmt.Sprintf("%q is not none, hourly, daily, weekly or monthly", name)) + } else if json.Unmarshal(o[name], &n) != nil || n < 0 || n > MaxKeep { + p.refuse("snapshots", raw, fmt.Sprintf("%s is not a whole number from 0 to %d", name, MaxKeep)) + } else { + k.set(name, n) + } + } + } + c.Snapshots = k + } + if len(p.out) > 0 { + return nil, p.out + } + return c, nil +} + +func clockForm(s string) string { + if !clock.MatchString(s) { + return "is not HH:MM, 00:00 to 23:59" + } + return "" +} + +func contains(l []string, s string) bool { + for _, x := range l { + if x == s { + return true + } + } + return false +} + +func (k *Keeps) set(kind string, n int) { + switch kind { + case "hourly": + k.Hourly = n + case "daily": + k.Daily = n + case "weekly": + k.Weekly = n + case "monthly": + k.Monthly = n + } +} + +// Of is the keep count of a kind. +func (k *Keeps) Of(kind string) int { + switch kind { + case "hourly": + return k.Hourly + case "daily": + return k.Daily + case "weekly": + return k.Weekly + } + return k.Monthly +} diff --git a/modules/zfs/cmd/zfs-tools/config_test.go b/modules/zfs/cmd/zfs-tools/config_test.go new file mode 100644 index 000000000..af1298096 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/config_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "strings" + "testing" +) + +// Design 42 zfs-owns-zfs §11: each value of a fixed form, not managed until set and not empty; a value of +// another form is refused before anything is written or run (§31, one table per value). + +func TestEachValueHasItsFormAndABlankOneIsNotManaged(t *testing.T) { + bad := func(key, val string) string { return `{"pool":"tank","` + key + `":` + val + `}` } + refused := []struct{ name, conf, why string }{ + {"pool with a space", `{"pool":"ta nk"}`, "pool's name"}, + {"pool with a newline", `{"pool":"tank\nx"}`, "pool's name"}, + {"pool not a string", `{"pool":3}`, "not a string"}, + {"arc-max below 64", bad("arc-max", "63"), "below 64"}, + {"arc-max at memory", bad("arc-max", "16000"), "not below the node's memory"}, + {"arc-max fraction", bad("arc-max", "100.5"), "whole number"}, + {"arc-max string", bad("arc-max", `"4096"`), "whole number"}, + {"arc-min above arc-max", `{"arc-max":1000,"arc-min":2000}`, "above arc-max"}, + {"module option of arc", bad("module-options", `["zfs_arc_max=1"]`), "arc-max and arc-min"}, + {"module option unknown", bad("module-options", `["zfs_foo=1"]`), "not a parameter"}, + {"module option not digits", bad("module-options", `["zfs_txg_timeout=5;x"]`), "not name=digits"}, + {"module option twice", bad("module-options", `["zfs_txg_timeout=5","zfs_txg_timeout=6"]`), "twice"}, + {"hostid short", bad("hostid", `"abc"`), "eight hexadecimal"}, + {"hostid zero", bad("hostid", `"00000000"`), "all zero"}, + {"hostid backtick", bad("hostid", "\"0123456`\""), "eight hexadecimal"}, + {"zed value", bad("zed", `"yes"`), "on, off"}, + {"zed handler", bad("zed-handlers", `["notify"]`), "not all-syslog"}, + {"trim schedule", bad("trim-schedule", `"daily"`), "none, weekly, monthly"}, + {"trim day", bad("trim-day", `"Monday"`), "Mon"}, + {"trim time", bad("trim-time", `"24:00"`), "HH:MM"}, + {"cache device with a slash", bad("cache-devices", `["a/b"]`), "under /dev/disk/by-id"}, + {"cache device with a dollar", bad("cache-devices", `["a$b"]`), "under /dev/disk/by-id"}, + {"cache device with a quote", bad("cache-devices", `["a\"b"]`), "under /dev/disk/by-id"}, + {"log device with a semicolon", bad("log-devices", `["a;b"]`), "under /dev/disk/by-id"}, + {"log mirror of one", `{"log-mirror":true,"log-devices":["a"]}`, "exactly two"}, + {"log mirror not a boolean", bad("log-mirror", `"yes"`), "true or false"}, + {"pool property", bad("pool-properties", `{"ashift":"12"}`), "not a pool property"}, + {"pool property value", bad("pool-properties", `{"failmode":"loud"}`), "failmode=loud"}, + {"dataset of another pool", bad("dataset-properties", `{"other/x":{"atime":"off"}}`), "not the pool's name"}, + {"dataset property sharenfs", bad("dataset-properties", `{"tank":{"sharenfs":"on"}}`), "not a dataset property"}, + {"dataset property sharesmb", bad("dataset-properties", `{"tank":{"sharesmb":"on"}}`), "not a dataset property"}, + {"dataset recordsize", bad("dataset-properties", `{"tank/a":{"recordsize":"100K"}}`), "power of two"}, + {"dataset compression", bad("dataset-properties", `{"tank/a":{"compression":"zstd-20"}}`), "compression"}, + {"dataset quota", bad("dataset-properties", `{"tank/a":{"quota":"5X"}}`), "quota"}, + {"dataset name with a snapshot mark", bad("dataset-properties", `{"tank/a@x":{"atime":"on"}}`), "dataset's name"}, + {"snapshots above the bound", bad("snapshots", `{"hourly":169}`), "0 to 168"}, + {"snapshots unknown kind", bad("snapshots", `{"yearly":1}`), "not none, hourly"}, + {"snapshots none with counts", bad("snapshots", `{"none":true,"daily":1}`), "none is given"}, + {"scrub cadence", bad("scrub-cadence", `"daily"`), "weekly, monthly"}, + {"not an object", `[1]`, "not a JSON object"}, + } + for _, c := range refused { + cfg, ref := ParseConfig([]byte(c.conf), 16000) + if cfg != nil || len(ref) == 0 || !strings.Contains(ref[0].Reason, c.why) { + t.Errorf("%s: %+v %+v, want a refusal saying %q", c.name, cfg, ref, c.why) + } + } + // Without the node's memory arc-max cannot be checked. + if _, ref := ParseConfig([]byte(`{"arc-max":1000}`), 0); len(ref) != 1 || ref[0].Key != "arc-max" { + t.Errorf("arc-max with no memory read: %v", ref) + } + // Absent, empty and null are not managed: no refusal, nothing set. + for _, conf := range []string{`{}`, `{"pool":"tank","cache-devices":[],"log-devices":[],"module-options":[],"zed-handlers":[], + "pool-properties":{},"dataset-properties":{},"snapshots":{},"hostid":"","zed":"","trim-schedule":"","arc-max":null}`} { + c, ref := ParseConfig([]byte(conf), 16000) + if len(ref) != 0 || c.ArcMax != nil || c.HostID != "" || c.Zed != "" || c.TrimSchedule != "" || c.Snapshots != nil || + len(c.CacheDevices)+len(c.LogDevices)+len(c.ModuleOptions)+len(c.ZedHandlers)+len(c.PoolProperties)+len(c.DatasetProperties) != 0 { + t.Errorf("%s: %+v %v", conf, c, ref) + } + } +} + +func TestAWholeConfigIsReadValueByValue(t *testing.T) { + c, ref := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly","scrub-day":"Mon","scrub-time":"00:00", + "arc-max":8192,"arc-min":1024,"module-options":["zfs_txg_timeout=10"],"hostid":"0A1B2C3D","zed":"on", + "zed-handlers":["all-syslog"],"trim-schedule":"weekly","trim-day":"Sun","trim-time":"03:00", + "cache-devices":["nvme-a","nvme-b"],"log-devices":["nvme-c","nvme-d"],"log-mirror":true, + "pool-properties":{"autotrim":"on","failmode":"default"}, + "dataset-properties":{"tank":{"compression":"zstd-3"},"tank/media":{"recordsize":"1M","quota":"5T"}}, + "snapshots":{"hourly":24,"daily":7}}`), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + if c.Pool != "tank" || *c.ArcMax != 8192 || *c.ArcMin != 1024 || c.HostID != "0a1b2c3d" || c.Zed != "on" || + c.ModuleOptions[0] != (ModuleOption{"zfs_txg_timeout", "10"}) || len(c.CacheDevices) != 2 || !*c.LogMirror || + c.PoolProperties["autotrim"] != "on" || c.DatasetProperties["tank/media"]["quota"] != "5T" || + c.Snapshots.Of("hourly") != 24 || c.Snapshots.Of("monthly") != 0 || c.Snapshots.None { + t.Errorf("%+v", c) + } + if c, _ := ParseConfig([]byte(`{"snapshots":{"none":true}}`), 0); !c.Snapshots.None { + t.Errorf("none: %+v", c.Snapshots) + } +} From 0fa4b5d8e73c05b5fe742c4ddfedb5518857251f Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:26:32 +0100 Subject: [PATCH 03/35] Z6 zfs-events-state: each change, refusal and wait said as its event and kept in the state applied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §25, §26, A-19 --- modules/zfs/cmd/zfs-tools/applied.go | 131 ++++++++++++++++++++++ modules/zfs/cmd/zfs-tools/applied_test.go | 66 +++++++++++ modules/zfs/cmd/zfs-tools/machine.go | 2 + 3 files changed, 199 insertions(+) create mode 100644 modules/zfs/cmd/zfs-tools/applied.go create mode 100644 modules/zfs/cmd/zfs-tools/applied_test.go diff --git a/modules/zfs/cmd/zfs-tools/applied.go b/modules/zfs/cmd/zfs-tools/applied.go new file mode 100644 index 000000000..b42fc4537 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/applied.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "fmt" + "time" +) + +// What the module says and keeps of what it changed (design 42 zfs-owns-zfs §25, §26). +// +// Each kind of event has a subject of its own on the module's events stream (the tool runner's or the +// process's `Emit`, by the kind's name). The state `applied` holds one key per config value, `{before, after, +// at, outcome}`, written by the apply and deleted by it when the value goes back to not managed, after its +// last `config.applied`. + +// The module's event kinds (§25), as the subject's last words. +const ( + EventApplied = "config.applied" + EventRefused = "config.refused" + EventWaiting = "config.waiting" + EventTakenPerson = "snapshot.taken.person" + EventTakenSchedule = "snapshot.taken.schedule" + EventRemovedPerson = "snapshot.removed.person" + EventRemovedSched = "snapshot.removed.schedule" + EventSkippedPerson = "snapshot.skipped.person" + EventSkippedSched = "snapshot.skipped.schedule" + EventRunSkipped = "run.skipped" + EventScrubRequested = ScrubEvent +) + +// The module's states (§26): the state's name, as `requires` declares it. +const ( + StateApplied = "applied" + StateSchedule = "schedule" + StateFindings = "findings" +) + +// The outcomes a key of the state `applied` holds. +const ( + OutcomeApplied = "applied" + OutcomeRefused = "refused" + OutcomeWaiting = "waiting" + OutcomeFailed = "failed" +) + +// StateStore is the module's states on the bus, as the process or the tools bundle reaches them. +type StateStore interface { + Put(state, key string, value any) error + Delete(state, key string) error + // Get answers false when the key is not there. + Get(state, key string) (json.RawMessage, bool, error) +} + +// AppliedEntry is one key of the state `applied`. +type AppliedEntry struct { + Before string `json:"before"` + After string `json:"after"` + At time.Time `json:"at"` + Outcome string `json:"outcome"` + // Why is the reason of a refusal or a failure, WaitsOn what a wait waits on, Since when it began. + Why string `json:"why,omitempty"` + WaitsOn string `json:"waits-on,omitempty"` + Since time.Time `json:"since,omitzero"` +} + +// say publishes one event of the module. +func (m *Machine) say(kind string, body map[string]any) error { + if m.Emit == nil { + return fmt.Errorf("this process publishes no events: %s was not said", kind) + } + return m.Emit(kind, body) +} + +// keep writes a key of the state `applied`. +func (m *Machine) keep(key string, e AppliedEntry) error { + if m.States == nil { + return fmt.Errorf("this process keeps no state: %s was not kept", key) + } + e.At = m.Now().UTC() + return m.States.Put(StateApplied, key, e) +} + +// Changed says a change as `config.applied` {target, key, before, after, read-back} and keeps it. The event +// goes first: the state is deleted only after the last event of its key. +func (m *Machine) Changed(target, key, before, after, readBack string) error { + if err := m.say(EventApplied, map[string]any{"target": target, "key": key, "before": before, "after": after, + "read-back": readBack}); err != nil { + return err + } + return m.keep(key, AppliedEntry{Before: before, After: after, Outcome: OutcomeApplied}) +} + +// Refused says a refusal as `config.refused` {key, value, reason} and keeps it, so zfs_health is unhealthy +// naming the key (§29). +func (m *Machine) Refused(r Refusal) error { + if err := m.say(EventRefused, map[string]any{"key": r.Key, "value": r.Value, "reason": r.Reason}); err != nil { + return err + } + return m.keep(r.Key, AppliedEntry{After: r.Value, Outcome: OutcomeRefused, Why: r.Reason}) +} + +// Waiting says a wait as `config.waiting` {key, waits-on, since} and keeps it; a wait already kept keeps +// its own since. +func (m *Machine) Waiting(key, waitsOn string) error { + since := m.Now().UTC() + if m.States != nil { + if raw, ok, err := m.States.Get(StateApplied, key); err == nil && ok { + var old AppliedEntry + if json.Unmarshal(raw, &old) == nil && old.Outcome == OutcomeWaiting && old.WaitsOn == waitsOn && !old.Since.IsZero() { + since = old.Since + } + } + } + if err := m.say(EventWaiting, map[string]any{"key": key, "waits-on": waitsOn, "since": since.Format(time.RFC3339)}); err != nil { + return err + } + return m.keep(key, AppliedEntry{Outcome: OutcomeWaiting, WaitsOn: waitsOn, Since: since}) +} + +// CommandFailed keeps a failed command under its key; the apply then exits non-zero (A-8). +func (m *Machine) CommandFailed(key string, cause error) error { + return m.keep(key, AppliedEntry{Outcome: OutcomeFailed, Why: cause.Error()}) +} + +// NotManaged is a value gone back to not managed: its key leaves the state, after its last `config.applied`. +func (m *Machine) NotManaged(key string) error { + if m.States == nil { + return nil + } + return m.States.Delete(StateApplied, key) +} diff --git a/modules/zfs/cmd/zfs-tools/applied_test.go b/modules/zfs/cmd/zfs-tools/applied_test.go new file mode 100644 index 000000000..d4ac4c462 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/applied_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +type memStates map[string]json.RawMessage + +func (s memStates) Put(state, key string, v any) error { + b, _ := json.Marshal(v) + s[state+"/"+key] = b + return nil +} +func (s memStates) Delete(state, key string) error { delete(s, state+"/"+key); return nil } +func (s memStates) Get(state, key string) (json.RawMessage, bool, error) { + b, ok := s[state+"/"+key] + return b, ok, nil +} + +// §25, §26: each change emits its event with before and after and keeps its key of the state `applied`, a +// refusal and a wait too, a wait keeping its first since, and a value back to not managed leaves the state. +func TestEachChangeIsSaidAndKeptAndANotManagedValueLeavesTheState(t *testing.T) { + f := newFake(t) + m := f.machine() + st := memStates{} + m.States = st + if err := m.Changed("pool", "pool-properties.autotrim", "off", "on", "on"); err != nil { + t.Fatal(err) + } + e := f.emitted[0] + if e["_type"] != "config.applied" || e["target"] != "pool" || e["key"] != "pool-properties.autotrim" || + e["before"] != "off" || e["after"] != "on" || e["read-back"] != "on" { + t.Errorf("event: %v", e) + } + var got AppliedEntry + json.Unmarshal(st["applied/pool-properties.autotrim"], &got) + if got.Before != "off" || got.After != "on" || got.Outcome != OutcomeApplied || !got.At.Equal(f.now.UTC()) { + t.Errorf("state: %+v", got) + } + if err := m.Refused(Refusal{Key: "hostid", Value: "zzz", Reason: "is not eight hexadecimal digits"}); err != nil { + t.Fatal(err) + } + if f.emitted[1]["_type"] != "config.refused" || f.emitted[1]["reason"] == "" || !strings.Contains(string(st["applied/hostid"]), `"refused"`) { + t.Errorf("refusal: %v %s", f.emitted[1], st["applied/hostid"]) + } + m.Waiting("log-devices", "the pool's scrub") + first := f.now + f.now = f.now.Add(time.Hour) + m.Waiting("log-devices", "the pool's scrub") + var w AppliedEntry + json.Unmarshal(st["applied/log-devices"], &w) + if f.emitted[3]["_type"] != "config.waiting" || !w.Since.Equal(first.UTC()) || !w.At.Equal(f.now.UTC()) || w.Outcome != OutcomeWaiting { + t.Errorf("wait: %v %+v", f.emitted[3], w) + } + m.NotManaged("log-devices") + if _, ok := st["applied/log-devices"]; ok { + t.Error("the key stays after the value went back to not managed") + } + m.Emit = nil + if err := m.Changed("pool", "k", "a", "b", "b"); err == nil { + t.Error("an event that cannot be said is said as done") + } +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 234523ed6..b70fa3f4a 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -113,6 +113,8 @@ type Machine struct { ReadFile func(path string) ([]byte, error) // Emit publishes an event as this module (the record of an act); nil where nothing can be published. Emit func(eventType string, body any) error + // States is the module's states on the bus (§26); nil where this process reaches none. + States StateStore // Errors is what the health check last saw of each device's error counts; nil judges every count // as past. Errors *ErrorMemory From a10bf45ed934a123f0a95e8e52b39dda455c6b8a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:27:31 +0100 Subject: [PATCH 04/35] Z7 zfs-apply-properties: pool and dataset properties converged, read back and said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §10, §16 --- modules/zfs/cmd/zfs-tools/apply_properties.go | 176 ++++++++++++++++++ .../cmd/zfs-tools/apply_properties_test.go | 100 ++++++++++ modules/zfs/cmd/zfs-tools/manifest_test.go | 12 +- 3 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 modules/zfs/cmd/zfs-tools/apply_properties.go create mode 100644 modules/zfs/cmd/zfs-tools/apply_properties_test.go diff --git a/modules/zfs/cmd/zfs-tools/apply_properties.go b/modules/zfs/cmd/zfs-tools/apply_properties.go new file mode 100644 index 000000000..495bf4ffa --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply_properties.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" +) + +// The apply's pool and dataset properties (design 42 zfs-owns-zfs §16, §10). +// +// For each set pool property the apply runs `zpool set = `, and for `default` `zpool +// set = `. For each set dataset property it runs `zfs set = +// `, and for `default` `zfs inherit `, so its source stays default. A property +// already at its value runs nothing. Each change is read back with `zpool get -Hp` or `zfs get -Hp` and said. +// sharenfs and sharesmb are never here: the parse refuses them (config.go). + +// poolDefaults are ZFS's defaults of the pool properties the module sets. +var poolDefaults = map[string]string{"autotrim": "off", "autoexpand": "off", "autoreplace": "off", "failmode": "wait"} + +// PoolGetArgv reads one pool property's value. +func PoolGetArgv(pool, property string) []string { + return []string{"get", "-Hp", "-o", "value", property, "--", pool} +} + +// PoolSetArgv sets one pool property. +func PoolSetArgv(pool, property, value string) []string { + return []string{"set", property + "=" + value, "--", pool} +} + +// DatasetGetArgv reads one dataset property's value and source. +func DatasetGetArgv(dataset, property string) []string { + return []string{"get", "-Hp", "-o", "value,source", property, "--", dataset} +} + +// DatasetSetArgv sets one dataset property. +func DatasetSetArgv(dataset, property, value string) []string { + return []string{"set", property + "=" + value, "--", dataset} +} + +// DatasetInheritArgv gives a dataset property back to ZFS's inheritance. +func DatasetInheritArgv(dataset, property string) []string { + return []string{"inherit", property, "--", dataset} +} + +// sizeBytes reads `1M`, `128K`, `5T` or plain digits as bytes; false for another form. +func sizeBytes(v string) (uint64, bool) { + mult := uint64(1) + if n := len(v); n > 0 { + if i := strings.IndexByte("KMGT", v[n-1]); i >= 0 { + mult = uint64(1) << (10 * (i + 1)) + v = v[:n-1] + } + } + n, err := strconv.ParseUint(v, 10, 64) + return n * mult, err == nil +} + +// sameValue says whether ZFS's `-p` value is the wanted one in the form the config states it. +func sameValue(property, got, want string) bool { + switch property { + case "recordsize", "quota", "reservation": + if want == "none" { + return got == "0" || got == "none" + } + g, ok1 := sizeBytes(got) + w, ok2 := sizeBytes(want) + return ok1 && ok2 && g == w + } + return got == want +} + +// ApplyProperties converges the set pool and dataset properties. It answers every command that failed, as +// one error; a property whose read-back differs from what was set is a failure too. +func (m *Machine) ApplyProperties(ctx context.Context, c *Config) error { + var failed []error + fail := func(key string, err error) { + failed = append(failed, fmt.Errorf("%s: %v", key, err)) + _ = m.CommandFailed(key, err) + } + for _, p := range sortedKeys(c.PoolProperties) { + key := "pool-properties." + p + want := c.PoolProperties[p] + if want == "default" { + want = poolDefaults[p] + } + got, err := m.ask(ctx, "zpool", PoolGetArgv(c.Pool, p)...) + if err != nil { + fail(key, err) + continue + } + before := strings.TrimSpace(got) + if before == want { + _ = m.keepAsSet(key, before) + continue + } + if _, err := m.AsRoot(ctx, "zpool", PoolSetArgv(c.Pool, p, want)...); err != nil { + fail(key, err) + continue + } + back, err := m.ask(ctx, "zpool", PoolGetArgv(c.Pool, p)...) + if err != nil || strings.TrimSpace(back) != want { + fail(key, fmt.Errorf("read back %q after setting %q (%v)", strings.TrimSpace(back), want, err)) + continue + } + if err := m.Changed("pool", key, before, want, strings.TrimSpace(back)); err != nil { + fail(key, err) + } + } + for _, ds := range sortedKeys(c.DatasetProperties) { + for _, p := range sortedKeys(c.DatasetProperties[ds]) { + key := "dataset-properties." + ds + "." + p + want := c.DatasetProperties[ds][p] + value, source, err := m.datasetProperty(ctx, ds, p) + if err != nil { + fail(key, err) + continue + } + before := value + if want == "default" { + if source != "local" && source != "received" { + _ = m.keepAsSet(key, before) + continue + } + if _, err := m.AsRoot(ctx, "zfs", DatasetInheritArgv(ds, p)...); err != nil { + fail(key, err) + continue + } + back, bsource, err := m.datasetProperty(ctx, ds, p) + if err != nil || bsource == "local" || bsource == "received" { + fail(key, fmt.Errorf("read back source %q after inheriting (%v)", bsource, err)) + continue + } + if err := m.Changed("dataset", key, before, "default", back); err != nil { + fail(key, err) + } + continue + } + if sameValue(p, value, want) { + _ = m.keepAsSet(key, before) + continue + } + if _, err := m.AsRoot(ctx, "zfs", DatasetSetArgv(ds, p, want)...); err != nil { + fail(key, err) + continue + } + back, _, err := m.datasetProperty(ctx, ds, p) + if err != nil || !sameValue(p, back, want) { + fail(key, fmt.Errorf("read back %q after setting %q (%v)", back, want, err)) + continue + } + if err := m.Changed("dataset", key, before, want, back); err != nil { + fail(key, err) + } + } + } + return errors.Join(failed...) +} + +// keepAsSet keeps a value found already at what is set: no command, no event, the state says it is as set. +func (m *Machine) keepAsSet(key, value string) error { + return m.keep(key, AppliedEntry{Before: value, After: value, Outcome: OutcomeApplied}) +} + +func (m *Machine) datasetProperty(ctx context.Context, ds, p string) (value, source string, err error) { + out, err := m.ask(ctx, "zfs", DatasetGetArgv(ds, p)...) + if err != nil { + return "", "", err + } + f := strings.Split(strings.TrimSpace(out), "\t") + if len(f) != 2 { + return "", "", fmt.Errorf("zfs get answered %q, not a value and a source", strings.TrimSpace(out)) + } + return f[0], f[1], nil +} diff --git a/modules/zfs/cmd/zfs-tools/apply_properties_test.go b/modules/zfs/cmd/zfs-tools/apply_properties_test.go new file mode 100644 index 000000000..12b897e2c --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply_properties_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "strings" + "testing" +) + +// §16, §31 convergence table: a property at its value runs nothing; a set one is set and read back and +// said; `default` is ZFS's default for a pool property and an inherit for a dataset's; a read-back that +// differs is a failure. +func TestPropertiesConvergeFromWhatZFSSays(t *testing.T) { + cfg := func(j string) *Config { + c, ref := ParseConfig([]byte(j), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + return c + } + type step struct{ cmd, out string } + cases := []struct { + name string + conf string + reads []step // read answers + writes []string + after map[string]string // a read that changes once the write ran + events int + fails bool + }{ + {"pool property at its value", `{"pool":"tank","pool-properties":{"autotrim":"on"}}`, + []step{{"zpool get -Hp -o value autotrim -- tank", "on\n"}}, nil, nil, 0, false}, + {"pool property set", `{"pool":"tank","pool-properties":{"autotrim":"on"}}`, + []step{{"zpool get -Hp -o value autotrim -- tank", "off\n"}}, []string{"zpool set autotrim=on -- tank"}, + map[string]string{"zpool get -Hp -o value autotrim -- tank": "on\n"}, 1, false}, + {"pool property to ZFS's default", `{"pool":"tank","pool-properties":{"failmode":"default"}}`, + []step{{"zpool get -Hp -o value failmode -- tank", "continue\n"}}, []string{"zpool set failmode=wait -- tank"}, + map[string]string{"zpool get -Hp -o value failmode -- tank": "wait\n"}, 1, false}, + {"pool property read back differs", `{"pool":"tank","pool-properties":{"autotrim":"on"}}`, + []step{{"zpool get -Hp -o value autotrim -- tank", "off\n"}}, []string{"zpool set autotrim=on -- tank"}, nil, 0, true}, + {"dataset size at its value in bytes", `{"pool":"tank","dataset-properties":{"tank/a":{"recordsize":"1M"}}}`, + []step{{"zfs get -Hp -o value,source recordsize -- tank/a", "1048576\tlocal\n"}}, nil, nil, 0, false}, + {"dataset compression set", `{"pool":"tank","dataset-properties":{"tank":{"compression":"zstd-3"}}}`, + []step{{"zfs get -Hp -o value,source compression -- tank", "lz4\tlocal\n"}}, []string{"zfs set compression=zstd-3 -- tank"}, + map[string]string{"zfs get -Hp -o value,source compression -- tank": "zstd-3\tlocal\n"}, 1, false}, + {"dataset default of an inherited one", `{"pool":"tank","dataset-properties":{"tank/a":{"atime":"default"}}}`, + []step{{"zfs get -Hp -o value,source atime -- tank/a", "on\tinherited from tank\n"}}, nil, nil, 0, false}, + {"dataset default of a local one", `{"pool":"tank","dataset-properties":{"tank/a":{"atime":"default"}}}`, + []step{{"zfs get -Hp -o value,source atime -- tank/a", "off\tlocal\n"}}, []string{"zfs inherit atime -- tank/a"}, + map[string]string{"zfs get -Hp -o value,source atime -- tank/a": "on\tdefault\n"}, 1, false}, + {"quota none at zero", `{"pool":"tank","dataset-properties":{"tank/a":{"quota":"none"}}}`, + []step{{"zfs get -Hp -o value,source quota -- tank/a", "0\tdefault\n"}}, nil, nil, 0, false}, + } + for _, c := range cases { + f := newFake(t) + f.uid = 0 + for _, r := range c.reads { + f.on(r.cmd, r.out) + } + wrote := map[string]bool{} + for _, w := range c.writes { + w := w + f.on(w, "") + f.after[w] = func(f *fake) { + wrote[w] = true + for k, v := range c.after { + f.on(k, v) + } + } + } + m := f.machine() + m.States = memStates{} + err := m.ApplyProperties(context.Background(), cfg(c.conf)) + if (err != nil) != c.fails { + t.Errorf("%s: error %v", c.name, err) + } + for _, w := range c.writes { + if !wrote[w] { + t.Errorf("%s: %q not run; ran %v", c.name, w, f.calls) + } + } + for _, call := range f.calls { + if (strings.Contains(call, " set ") || strings.Contains(call, " inherit ")) && !contains(c.writes, call) { + t.Errorf("%s: ran %q", c.name, call) + } + } + if len(f.emitted) != c.events { + t.Errorf("%s: %d events, want %d: %v", c.name, len(f.emitted), c.events, f.emitted) + } + } +} + +// §10: a dataset's sharenfs and sharesmb are never converged, never run: the parse refuses them, so the +// apply has no Config to run. +func TestSharenfsAndSharesmbAreNeverASetProperty(t *testing.T) { + for _, p := range []string{"sharenfs", "sharesmb"} { + if c, ref := ParseConfig([]byte(`{"pool":"tank","dataset-properties":{"tank":{"`+p+`":"on"}}}`), 16000); c != nil || len(ref) == 0 { + t.Errorf("%s accepted", p) + } + } +} diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index b917c57a1..3814ce6ae 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -218,12 +218,22 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"}, "ScrubArgv": {"zpool", "scrub"}, "WaitArgv": {"zpool", "wait"}, "TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"}, + "PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"}, + "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), "PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(), - "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank")} + "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"), + "PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"), + "DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"), + "DatasetInheritArgv": DatasetInheritArgv("tank", "atime")} for fn, argv := range verbs { + for _, a := range argv { + if fn != "FollowArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") { + t.Errorf("%s runs %q", fn, a) + } + } if argv[0] != allowed[fn][1] { t.Errorf("%s runs %q, not %q", fn, argv[0], allowed[fn][1]) } From 2fb1e5c4eaa37408535a9c1bf92e294a93dd9423 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:32:06 +0100 Subject: [PATCH 05/35] Z8 zfs-apply-devices: cache and log devices converged by their stable names, after their checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §17 --- modules/zfs/cmd/zfs-tools/apply_devices.go | 516 ++++++++++++++++++ .../zfs/cmd/zfs-tools/apply_devices_test.go | 240 ++++++++ modules/zfs/cmd/zfs-tools/helpers_test.go | 9 +- modules/zfs/cmd/zfs-tools/machine.go | 5 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 9 +- 5 files changed, 775 insertions(+), 4 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/apply_devices.go create mode 100644 modules/zfs/cmd/zfs-tools/apply_devices_test.go diff --git a/modules/zfs/cmd/zfs-tools/apply_devices.go b/modules/zfs/cmd/zfs-tools/apply_devices.go new file mode 100644 index 000000000..191d0d2df --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply_devices.go @@ -0,0 +1,516 @@ +package main + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" +) + +// The apply's cache and log devices (design 42 zfs-owns-zfs §17). +// +// From `zpool status -P -L ` the apply knows each entry's kernel path, from `zpool status -P ` the +// name the pool holds it by, and from the link under /dev/disk/by-id each declared name's kernel path. Then, in +// this order: an entry named by its kernel name that is a declared device is removed and added again by its +// stable name; a declared device not in the pool is added after its checks, never with -f; only while the list +// is set and not empty is a cache entry it does not name removed; and a log vdev it does not name, one to +// rename, or the log when `log-mirror` changes is removed only while the pool is healthy and idle, else it is +// a wait. A declared device that is not present is a wait. + +// ByIDDir is where a declared device's stable name is a link. +const ByIDDir = "/dev/disk/by-id/" + +// DeviceStatusArgv reads one pool's status with the paths its vdevs are held by (real false) or the kernel +// paths they resolve to (real true). +func DeviceStatusArgv(pool string, real bool) []string { + a := []string{"status", "-P"} + if real { + a = append(a, "-L") + } + return append(a, "--", pool) +} + +// AllRealStatusArgv reads every imported pool's vdevs by kernel path. +func AllRealStatusArgv() []string { return []string{"status", "-P", "-L"} } + +// RemoveArgv removes one cache or log entry of the pool. +func RemoveArgv(pool, entry string) []string { return []string{"remove", "--", pool, entry} } + +// AddArgv adds cache or log devices by their paths; the log as one mirror when mirror is true. Never -f. +func AddArgv(pool, role string, mirror bool, paths ...string) []string { + a := []string{"add", "--", pool, role} + if mirror { + a = append(a, "mirror") + } + return append(a, paths...) +} + +// WipefsArgv lists a device's signatures and writes nothing. +func WipefsArgv(device string) []string { return []string{"--no-act", device} } + +// ZdbLabelArgv reads a device's ZFS label. +func ZdbLabelArgv(device string) []string { return []string{"-l", device} } + +// devEntry is one cache or log device of the pool. +type devEntry struct { + Held string // the path the pool holds it by + Real string // its kernel path + Role string // cache or log + Vdev string // the top-level vdev it is in ("" for a device of its own) +} + +// logVdev is one top-level vdev of the log: a device of its own, or a mirror. +type logVdev struct { + Name string + Mirror bool + Members []devEntry +} + +// poolDevices is what the apply knows of the pool's cache and log. +type poolDevices struct { + Cache []devEntry + Log []logVdev + Healthy bool // ONLINE, no error on any device, no scrub or resilver running + Why string // what holds it from being healthy +} + +func (p *poolDevices) members() []devEntry { + out := append([]devEntry{}, p.Cache...) + for _, v := range p.Log { + out = append(out, v.Members...) + } + return out +} + +func (p *poolDevices) inPool(real string) *devEntry { + for _, e := range p.members() { + if e.Real == real { + return &e + } + } + return nil +} + +func findPool(out, pool string) (*PoolStatus, error) { + ps, err := ParseStatus(out) + if err != nil { + return nil, err + } + for i := range ps { + if ps[i].Name == pool { + return &ps[i], nil + } + } + return nil, fmt.Errorf("zpool status does not show the pool %s", pool) +} + +func sectionOf(p *PoolStatus, role string) *Device { + if p.Root == nil { + return nil + } + for _, c := range p.Root.Children { + if c.Kind == "section" && c.Role == role { + return c + } + } + return nil +} + +func deviceErrors(d *Device) uint64 { + n := d.Errors() + for _, c := range d.Children { + n += deviceErrors(c) + } + return n +} + +// devicesOf reads the pool's cache and log, held-by and real read side by side. +func (m *Machine) devicesOf(ctx context.Context, pool string) (*poolDevices, error) { + var st [2]*PoolStatus + for i, real := range []bool{false, true} { + out, err := m.ask(ctx, "zpool", DeviceStatusArgv(pool, real)...) + if err != nil { + return nil, err + } + if st[i], err = findPool(out, pool); err != nil { + return nil, err + } + } + held, real := st[0], st[1] + pd := &poolDevices{Healthy: true} + switch { + case real.State != "ONLINE": + pd.Healthy, pd.Why = false, "the pool is "+real.State + case real.Root != nil && deviceErrors(real.Root) > 0: + pd.Healthy, pd.Why = false, "a device of the pool has read, write or checksum errors" + } + for _, s := range real.Scans { + if (s.Kind == "scrub" || s.Kind == "resilver") && (s.State == "running" || s.State == "paused") { + pd.Healthy, pd.Why = false, "a "+s.Kind+" is running on the pool" + } + } + for _, role := range []string{"cache", "log"} { + hs, rs := sectionOf(held, role), sectionOf(real, role) + if hs == nil || rs == nil { + continue + } + if len(hs.Children) != len(rs.Children) { + return nil, fmt.Errorf("zpool status -P and -P -L list a different %s", role) + } + for i, hc := range hs.Children { + rc := rs.Children[i] + if len(hc.Children) != len(rc.Children) { + return nil, fmt.Errorf("zpool status -P and -P -L list a different %s", role) + } + if len(hc.Children) == 0 { + e := devEntry{Held: hc.Name, Real: rc.Name, Role: role} + if role == "cache" { + pd.Cache = append(pd.Cache, e) + } else { + pd.Log = append(pd.Log, logVdev{Name: hc.Name, Members: []devEntry{e}}) + } + continue + } + v := logVdev{Name: hc.Name, Mirror: true} + for j, hm := range hc.Children { + v.Members = append(v.Members, devEntry{Held: hm.Name, Real: rc.Children[j].Name, Role: role, Vdev: hc.Name}) + } + if role == "log" { + pd.Log = append(pd.Log, v) + } else { + return nil, fmt.Errorf("the cache holds a vdev %s, which a cache never is", hc.Name) + } + } + } + return pd, nil +} + +// declared is one device the config names. +type declared struct { + Name, Path, Real, Role string + Present bool +} + +func (m *Machine) resolveDeclared(names []string, role string) []declared { + var out []declared + for _, n := range names { + d := declared{Name: n, Path: ByIDDir + n, Role: role} + if m.Resolve != nil { + if r, err := m.Resolve(d.Path); err == nil { + d.Real, d.Present = r, true + } + } + out = append(out, d) + } + return out +} + +func byID(held string) bool { return strings.HasPrefix(held, ByIDDir) } + +var labelName = regexp.MustCompile(`(?m)^\s*name:\s*'([^']*)'`) + +// partOf says whether entry is device itself or one of its partitions. +func partOf(entry, device string) bool { + rest, ok := strings.CutPrefix(entry, device) + if !ok { + return false + } + if rest == "" { + return true + } + rest = strings.TrimPrefix(rest, "p") + return rest != "" && strings.Trim(rest, "0123456789") == "" +} + +// checkDevice is the refusal's reason for a device to add, "" when it may be added (§17): a partition table or +// a file-system signature refuses it, but for a zfs_member signature judged by its label (this pool's is +// allowed, another's refused), and a member of another imported pool is refused. +func (m *Machine) checkDevice(ctx context.Context, pool string, d declared, others []string) (string, error) { + for _, o := range others { + if partOf(o, d.Real) || partOf(d.Real, o) { + return d.Name + " is a member of another imported pool", nil + } + } + r, err := m.AsRoot(ctx, "wipefs", WipefsArgv(d.Path)...) + if err != nil { + return "", err + } + zfsMember := false + for _, l := range lines(r.Out) { + f := strings.Fields(l) + if len(f) == 0 || f[0] == "DEVICE" { + continue + } + if len(f) < 3 { + return d.Name + " shows a signature wipefs names no type for: " + l, nil + } + if f[2] != "zfs_member" { + return fmt.Sprintf("%s holds a %s signature or partition table", d.Name, f[2]), nil + } + zfsMember = true + } + if !zfsMember { + return "", nil + } + z, err := m.AsRoot(ctx, "zdb", ZdbLabelArgv(d.Path)...) + if err != nil { + return d.Name + " holds a ZFS label that cannot be read: " + err.Error(), nil + } + lm := labelName.FindStringSubmatch(z.Out) + switch { + case lm == nil: + return d.Name + " holds a ZFS label that names no pool", nil + case lm[1] != pool: + return fmt.Sprintf("%s holds the label of the pool %s, not %s", d.Name, lm[1], pool), nil + } + return "", nil +} + +// otherPoolDevices are the kernel paths of every imported pool's vdevs but the pool's own. +func (m *Machine) otherPoolDevices(ctx context.Context, pool string) ([]string, error) { + out, err := m.ask(ctx, "zpool", AllRealStatusArgv()...) + if err != nil { + return nil, err + } + ps, err := ParseStatus(out) + if err != nil { + return nil, err + } + var paths []string + var walk func(d *Device) + walk = func(d *Device) { + for _, c := range d.Children { + if strings.HasPrefix(c.Name, "/dev/") { + paths = append(paths, c.Name) + } + walk(c) + } + } + for _, p := range ps { + if p.Name != pool && p.Root != nil { + walk(p.Root) + } + } + return paths, nil +} + +// ApplyDevices converges the cache and log devices the config names; it answers every failed command as one +// error. An empty list is not managed, so it removes nothing. +func (m *Machine) ApplyDevices(ctx context.Context, c *Config) error { + if c.Pool == "" || (len(c.CacheDevices) == 0 && len(c.LogDevices) == 0) { + return nil + } + var failed []error + fail := func(key string, err error) { + failed = append(failed, fmt.Errorf("%s: %v", key, err)) + _ = m.CommandFailed(key, err) + } + others, err := m.otherPoolDevices(ctx, c.Pool) + if err != nil { + fail("cache-devices", err) + return errors.Join(failed...) + } + for _, role := range []string{"cache", "log"} { + names := c.CacheDevices + if role == "log" { + names = c.LogDevices + } + if len(names) == 0 { + continue + } + key := role + "-devices" + if err := m.applyRole(ctx, c, role, key, m.resolveDeclared(names, role), others); err != nil { + fail(key, err) + } + } + return errors.Join(failed...) +} + +func (m *Machine) applyRole(ctx context.Context, c *Config, role, key string, want []declared, others []string) error { + pd, err := m.devicesOf(ctx, c.Pool) + if err != nil { + return err + } + before := m.rolesSaid(pd, role) + for _, d := range want { + if !d.Present { + return m.Waiting(key, "the device "+d.Name+" to appear under "+ByIDDir) + } + } + if role == "log" { + if bad, _ := badLog(c, want, pd); len(bad) > 0 && !pd.Healthy { + return m.Waiting(key, "the pool to be ONLINE without errors and with no scrub or resilver ("+pd.Why+")") + } + } + // Checks first, so nothing is removed that cannot be put back. + for _, d := range want { + if e := pd.inPool(d.Real); e != nil && byID(e.Held) { + continue + } + var out []string + if pd.inPool(d.Real) == nil { + out = others + } + why, err := m.checkDevice(ctx, c.Pool, d, out) + if err != nil { + return err + } + if why != "" { + return m.Refused(Refusal{Key: key, Value: d.Name, Reason: why}) + } + } + var changes []string + if role == "cache" { + changes, err = m.applyCache(ctx, c, want, pd) + } else { + changes, err = m.applyLog(ctx, c, want, pd) + } + if err != nil { + return err + } + after, err := m.devicesOf(ctx, c.Pool) + if err != nil { + return err + } + for _, d := range want { + if e := after.inPool(d.Real); e == nil || !byID(e.Held) { + if len(changes) == 0 { + continue + } + return fmt.Errorf("read back: %s is not in the pool by its stable name after the change", d.Name) + } + } + if len(changes) == 0 { + return m.keepAsSet(key, before) + } + return m.Changed("pool", key, before, strings.Join(changes, "; "), m.rolesSaid(after, role)) +} + +func (m *Machine) rolesSaid(pd *poolDevices, role string) string { + var held []string + for _, e := range pd.members() { + if e.Role == role { + held = append(held, e.Held) + } + } + return strings.Join(held, " ") +} + +func (m *Machine) applyCache(ctx context.Context, c *Config, want []declared, pd *poolDevices) ([]string, error) { + var changes []string + for _, d := range want { + if e := pd.inPool(d.Real); e != nil && e.Role == "cache" && !byID(e.Held) { + if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, e.Held)...); err != nil { + return changes, err + } + if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "cache", false, d.Path)...); err != nil { + return changes, err + } + changes = append(changes, fmt.Sprintf("%s re-added by its stable name %s", e.Held, d.Name)) + } + } + var adds []string + var names []string + for _, d := range want { + if pd.inPool(d.Real) == nil { + adds, names = append(adds, d.Path), append(names, d.Name) + } + } + if len(adds) > 0 { + if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "cache", false, adds...)...); err != nil { + return changes, err + } + changes = append(changes, "added "+strings.Join(names, " ")) + } + for _, e := range pd.Cache { + if !declaredReal(want, e.Real) { + if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, e.Held)...); err != nil { + return changes, err + } + changes = append(changes, "removed "+e.Held) + } + } + return changes, nil +} + +func declaredReal(want []declared, real string) bool { + for _, d := range want { + if d.Real == real { + return true + } + } + return false +} + +// badLog is the log vdevs that must go: one holding a device the config does not name or one held by its +// kernel name, any when log-mirror asks the other form, and a mirror whose members are not the declared pair. +func badLog(c *Config, want []declared, pd *poolDevices) (bad []logVdev, mirrorChange bool) { + for _, v := range pd.Log { + b := false + for _, e := range v.Members { + if !declaredReal(want, e.Real) || !byID(e.Held) { + b = true + } + } + if c.LogMirror != nil { + if *c.LogMirror && (!v.Mirror || len(v.Members) != len(want)) { + b, mirrorChange = true, true + } + if !*c.LogMirror && v.Mirror { + b, mirrorChange = true, true + } + } + if b { + bad = append(bad, v) + } + } + return bad, mirrorChange +} + +func (m *Machine) applyLog(ctx context.Context, c *Config, want []declared, pd *poolDevices) ([]string, error) { + var changes []string + bad, mirrorChange := badLog(c, want, pd) + if len(bad) > 0 { + for _, v := range bad { + if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, v.Name)...); err != nil { + return changes, err + } + note := "removed " + v.Name + if mirrorChange { + note += " (a log-mirror change: a removal and a re-add)" + } + changes = append(changes, note) + } + var err error + if pd, err = m.devicesOf(ctx, c.Pool); err != nil { + return changes, err + } + } + var missing []declared + for _, d := range want { + if pd.inPool(d.Real) == nil { + missing = append(missing, d) + } + } + if len(missing) == 0 { + return changes, nil + } + var paths, names []string + for _, d := range missing { + paths, names = append(paths, d.Path), append(names, d.Name) + } + mirror := c.LogMirror != nil && *c.LogMirror + if mirror && len(missing) == len(want) { + if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "log", true, paths...)...); err != nil { + return changes, err + } + } else { + for _, p := range paths { + if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "log", false, p)...); err != nil { + return changes, err + } + } + } + return append(changes, "added "+strings.Join(names, " ")), nil +} diff --git a/modules/zfs/cmd/zfs-tools/apply_devices_test.go b/modules/zfs/cmd/zfs-tools/apply_devices_test.go new file mode 100644 index 000000000..7fc66d444 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply_devices_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "fmt" + "strings" + "testing" +) + +type devSpec struct { + state, scan string + errs bool + cache [][2]string // held, real + logs [][][2]string // top-level log vdevs; more than one member is a mirror +} + +func (s devSpec) text(real bool) string { + state, scan := s.state, s.scan + if state == "" { + state = "ONLINE" + } + if scan == "" { + scan = "scrub repaired 0B in 00:04:11 with 0 errors on Sun Jul 12 00:28:12 2026" + } + cksum := "0" + if s.errs { + cksum = "3" + } + name := func(p [2]string) string { + if real { + return p[1] + } + return p[0] + } + var b strings.Builder + fmt.Fprintf(&b, " pool: tank\n state: %s\n scan: %s\nconfig:\n\n\tNAME STATE READ WRITE CKSUM\n\ttank %s 0 0 %s\n", state, scan, state, cksum) + b.WriteString("\t /dev/disk/by-id/data-1 ONLINE 0 0 0\n") + if len(s.logs) > 0 { + b.WriteString("\tlogs\t\n") + for i, v := range s.logs { + if len(v) == 1 { + fmt.Fprintf(&b, "\t %s ONLINE 0 0 0\n", name(v[0])) + continue + } + fmt.Fprintf(&b, "\t mirror-%d ONLINE 0 0 0\n", i) + for _, m := range v { + fmt.Fprintf(&b, "\t %s ONLINE 0 0 0\n", name(m)) + } + } + } + if len(s.cache) > 0 { + b.WriteString("\tcache\t\n") + for _, c := range s.cache { + fmt.Fprintf(&b, "\t %s ONLINE 0 0 0\n", name(c)) + } + } + b.WriteString("\nerrors: No known data errors\n") + return b.String() +} + +func (s devSpec) set(f *fake) { + f.on("zpool status -P -- tank", s.text(false)) + f.on("zpool status -P -L -- tank", s.text(true)) +} + +func dev(n string) [2]string { return [2]string{"/dev/disk/by-id/" + n, "/dev/" + n + "k"} } +func kern(n string) [2]string { return [2]string{"/dev/" + n + "k", "/dev/" + n + "k"} } + +// §17, §31 convergence table: a kernel-named entry is re-added by its stable name; an add is after its +// checks, never -f; an empty list removes nothing; a log removal while not healthy is a wait; a log-mirror +// change is a removal and a re-add, said so. +func TestDevicesConvergeFromWhatZFSSays(t *testing.T) { + const wipeGPT = "DEVICE OFFSET TYPE UUID LABEL\nnvme1k 0x200 gpt\n" + const wipeZFS = "DEVICE OFFSET TYPE UUID LABEL\nnvme1k 0x4000 zfs_member 123 tank\n" + cases := []struct { + name string + conf string + before, mid devSpec + hasMid bool + after devSpec + answers map[string]string + others string + ran []string + notRan []string + event, state string + wantErr bool + waitsOrRefuse string + }{ + {name: "kernel name re-added by its stable name", conf: `{"pool":"tank","cache-devices":["c1"]}`, + before: devSpec{cache: [][2]string{kern("c1")}}, mid: devSpec{}, hasMid: true, after: devSpec{cache: [][2]string{dev("c1")}}, + answers: map[string]string{"zdb -l /dev/disk/by-id/c1": "name: 'tank'\n", "wipefs --no-act /dev/disk/by-id/c1": wipeZFS}, + ran: []string{"zpool remove -- tank /dev/c1k", "zpool add -- tank cache /dev/disk/by-id/c1"}, event: "config.applied", state: "applied"}, + {name: "declared device not in the pool is added", conf: `{"pool":"tank","cache-devices":["c2"]}`, + before: devSpec{}, after: devSpec{cache: [][2]string{dev("c2")}}, + answers: map[string]string{"wipefs --no-act /dev/disk/by-id/c2": ""}, + ran: []string{"zpool add -- tank cache /dev/disk/by-id/c2"}, event: "config.applied", state: "applied"}, + {name: "a partition table refuses it", conf: `{"pool":"tank","cache-devices":["c2"]}`, + before: devSpec{}, + answers: map[string]string{"wipefs --no-act /dev/disk/by-id/c2": wipeGPT}, + notRan: []string{"zpool add"}, event: "config.refused", state: "refused"}, + {name: "another pool's label refuses it", conf: `{"pool":"tank","cache-devices":["c2"]}`, + before: devSpec{}, + answers: map[string]string{"wipefs --no-act /dev/disk/by-id/c2": wipeZFS, "zdb -l /dev/disk/by-id/c2": "name: 'other'\n"}, + notRan: []string{"zpool add"}, event: "config.refused", state: "refused"}, + {name: "this pool's label allows it", conf: `{"pool":"tank","cache-devices":["c2"]}`, + before: devSpec{}, after: devSpec{cache: [][2]string{dev("c2")}}, + answers: map[string]string{"wipefs --no-act /dev/disk/by-id/c2": wipeZFS, "zdb -l /dev/disk/by-id/c2": "name: 'tank'\n"}, + ran: []string{"zpool add -- tank cache /dev/disk/by-id/c2"}, event: "config.applied", state: "applied"}, + {name: "a member of another imported pool is refused", conf: `{"pool":"tank","cache-devices":["c2"]}`, + before: devSpec{}, others: "/dev/c2kp1", notRan: []string{"wipefs", "zpool add"}, event: "config.refused", state: "refused"}, + {name: "a device not present is a wait", conf: `{"pool":"tank","cache-devices":["c9"]}`, + before: devSpec{}, notRan: []string{"zpool add", "wipefs"}, event: "config.waiting", state: "waiting"}, + {name: "an empty cache list removes nothing", conf: `{"pool":"tank","cache-devices":[],"log-devices":["l1"]}`, + before: devSpec{cache: [][2]string{dev("old")}, logs: [][][2]string{{dev("l1")}}}, notRan: []string{"zpool remove", "zpool add"}, state: "applied"}, + {name: "a cache entry not named is removed", conf: `{"pool":"tank","cache-devices":["c1"]}`, + before: devSpec{cache: [][2]string{dev("c1"), dev("old")}}, after: devSpec{cache: [][2]string{dev("c1")}}, + ran: []string{"zpool remove -- tank /dev/disk/by-id/old"}, event: "config.applied", state: "applied"}, + {name: "a log removal while a scrub runs is a wait", conf: `{"pool":"tank","log-devices":["l1"]}`, + before: devSpec{scan: "scrub in progress since Sun Jul 12 00:28:12 2026", logs: [][][2]string{{kern("l1")}}}, + notRan: []string{"zpool remove", "zpool add"}, event: "config.waiting", state: "waiting"}, + {name: "a log removal with errors is a wait", conf: `{"pool":"tank","log-devices":["l1"]}`, + before: devSpec{errs: true, logs: [][][2]string{{dev("l1"), dev("x")}}}, notRan: []string{"zpool remove"}, event: "config.waiting", state: "waiting"}, + {name: "a log-mirror change is a removal and a re-add", conf: `{"pool":"tank","log-devices":["l1","l2"],"log-mirror":true}`, + before: devSpec{logs: [][][2]string{{dev("l1")}, {dev("l2")}}}, mid: devSpec{}, hasMid: true, + after: devSpec{logs: [][][2]string{{dev("l1"), dev("l2")}}}, + answers: map[string]string{"wipefs --no-act /dev/disk/by-id/l1": "", "wipefs --no-act /dev/disk/by-id/l2": ""}, + ran: []string{"zpool remove -- tank /dev/disk/by-id/l1", "zpool remove -- tank /dev/disk/by-id/l2", + "zpool add -- tank log mirror /dev/disk/by-id/l1 /dev/disk/by-id/l2"}, event: "config.applied", state: "applied"}, + {name: "everything as set runs nothing", conf: `{"pool":"tank","cache-devices":["c1"],"log-devices":["l1","l2"],"log-mirror":true}`, + before: devSpec{cache: [][2]string{dev("c1")}, logs: [][][2]string{{dev("l1"), dev("l2")}}}, + notRan: []string{"zpool remove", "zpool add"}, state: "applied"}, + } + for _, c := range cases { + f := newFake(t) + f.uid = 0 + for _, n := range []string{"c1", "c2", "l1", "l2", "old", "x"} { + f.links["/dev/disk/by-id/"+n] = "/dev/" + n + "k" + } + c.before.set(f) + f.on("zpool status -P -L", " pool: other\n state: ONLINE\nconfig:\n\n\tNAME STATE READ WRITE CKSUM\n\tother ONLINE 0 0 0\n\t "+ + orDefault(c.others, "/dev/zzz")+" ONLINE 0 0 0\n\nerrors: No known data errors\n") + for k, v := range c.answers { + f.on(k, v) + } + mid := c.mid + if !c.hasMid { + mid = c.after + } + // A removal moves the pool to mid, an add to after. + hook := func(spec devSpec) func(*fake) { return func(f *fake) { spec.set(f) } } + for _, r := range c.ran { + f.on(r, "") + if strings.HasPrefix(r, "zpool remove") { + f.after[r] = hook(mid) + } else { + f.after[r] = hook(c.after) + } + } + m := f.machine() + st := memStates{} + m.States = st + cfg, ref := ParseConfig([]byte(c.conf), 16000) + if len(ref) != 0 { + t.Fatal(c.name, ref) + } + err := m.ApplyDevices(context.Background(), cfg) + if (err != nil) != c.wantErr { + t.Errorf("%s: error %v", c.name, err) + } + for _, r := range c.ran { + if len(ran(f, r)) != 1 { + t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls) + } + } + for _, n := range c.notRan { + if len(ran(f, n)) != 0 { + t.Errorf("%s: ran %v", c.name, ran(f, n)) + } + } + for _, call := range f.calls { + if strings.Contains(call, " -f") { + t.Errorf("%s: forced: %s", c.name, call) + } + } + if c.event == "" && len(f.emitted) != 0 { + t.Errorf("%s: events %v", c.name, f.emitted) + } + if c.event != "" && (len(f.emitted) != 1 || f.emitted[0]["_type"] != c.event) { + t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event) + } + if c.state != "" { + found := false + for k, v := range st { + if strings.Contains(string(v), `"outcome":"`+c.state+`"`) { + found = true + _ = k + } + } + if !found { + t.Errorf("%s: state %v, want outcome %s", c.name, st, c.state) + } + } + } +} + +func orDefault(s, d string) string { + if s == "" { + return d + } + return s +} + +// §17: a log-mirror change is said as a removal and a re-add. +func TestALogMirrorChangeIsSaidAsARemovalAndAReAdd(t *testing.T) { + f := newFake(t) + f.uid = 0 + for _, n := range []string{"l1", "l2"} { + f.links["/dev/disk/by-id/"+n] = "/dev/" + n + "k" + } + before := devSpec{logs: [][][2]string{{dev("l1")}, {dev("l2")}}} + after := devSpec{logs: [][][2]string{{dev("l1"), dev("l2")}}} + before.set(f) + f.on("zpool status -P -L", "") + f.on("wipefs --no-act /dev/disk/by-id/l1", "") + f.on("wipefs --no-act /dev/disk/by-id/l2", "") + f.on("zpool remove -- tank /dev/disk/by-id/l1", "") + f.on("zpool remove -- tank /dev/disk/by-id/l2", "") + f.after["zpool remove -- tank /dev/disk/by-id/l2"] = func(f *fake) { devSpec{}.set(f) } + f.on("zpool add -- tank log mirror /dev/disk/by-id/l1 /dev/disk/by-id/l2", "") + f.after["zpool add -- tank log mirror /dev/disk/by-id/l1 /dev/disk/by-id/l2"] = func(f *fake) { after.set(f) } + m := f.machine() + m.States = memStates{} + cfg, _ := ParseConfig([]byte(`{"pool":"tank","log-devices":["l1","l2"],"log-mirror":true}`), 16000) + if err := m.ApplyDevices(context.Background(), cfg); err != nil { + t.Fatal(err) + } + if a, _ := f.emitted[0]["after"].(string); !strings.Contains(a, "a log-mirror change: a removal and a re-add") { + t.Errorf("event %v", f.emitted[0]) + } +} diff --git a/modules/zfs/cmd/zfs-tools/helpers_test.go b/modules/zfs/cmd/zfs-tools/helpers_test.go index 3cc1be180..9a245c587 100644 --- a/modules/zfs/cmd/zfs-tools/helpers_test.go +++ b/modules/zfs/cmd/zfs-tools/helpers_test.go @@ -32,13 +32,14 @@ type fake struct { emitted []map[string]any emitErr error files map[string]string + links map[string]string now time.Time uid int errors *ErrorMemory } func newFake(t *testing.T) *fake { - return &fake{t: t, answers: map[string]Ran{}, after: map[string]func(*fake){}, files: map[string]string{}, + return &fake{t: t, answers: map[string]Ran{}, after: map[string]func(*fake){}, files: map[string]string{}, links: map[string]string{}, now: time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local), uid: 1000, errors: NewErrorMemory()} } @@ -74,6 +75,12 @@ func (f *fake) machine() *Machine { } return []byte(s), nil }, + Resolve: func(p string) (string, error) { + if r, ok := f.links[p]; ok { + return r, nil + } + return "", os.ErrNotExist + }, Errors: f.errors, Emit: func(eventType string, body any) error { if f.emitErr != nil { diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index b70fa3f4a..3164bfacc 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "syscall" @@ -111,6 +112,8 @@ type Machine struct { Now func() time.Time // ReadFile reads a kernel file (the ARC's counters). ReadFile func(path string) ([]byte, error) + // Resolve follows a link to its kernel path (a declared device's name under /dev/disk/by-id). + Resolve func(path string) (string, error) // Emit publishes an event as this module (the record of an act); nil where nothing can be published. Emit func(eventType string, body any) error // States is the module's states on the bus (§26); nil where this process reaches none. @@ -122,7 +125,7 @@ type Machine struct { // Here is the machine this process runs on. func Here() *Machine { - return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Errors: LoadErrorMemory(ErrorsFile)} + return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Errors: LoadErrorMemory(ErrorsFile)} } // NotApplicable is the answer of a machine without ZFS: the module does not apply here. diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 3814ce6ae..eca905d50 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -220,6 +220,8 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"}, "PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"}, "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, + "DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"}, + "AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), @@ -227,7 +229,10 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"), "PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"), "DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"), - "DatasetInheritArgv": DatasetInheritArgv("tank", "atime")} + "DatasetInheritArgv": DatasetInheritArgv("tank", "atime"), + "DeviceStatusArgv": DeviceStatusArgv("tank", true), "AllRealStatusArgv": AllRealStatusArgv(), + "RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"), + "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x")} for fn, argv := range verbs { for _, a := range argv { if fn != "FollowArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") { @@ -248,7 +253,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { } } call := regexp.MustCompile(`(\.(ask|AsRoot|Escalated|Run|Start|Output|CombinedOutput)\(|exec\.Command(Context)?\(|os\.StartProcess\(|syscall\.(Exec|ForkExec)\()`) - site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl)", (\w+)\(`) + site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb)", (\w+)\(`) // The bodies that run what they are given, each once. bodies := map[string]bool{ "machine.go: r := m.Run(ctx, name, args...)": true, From 1bea6edcb88d8ca5b2187881e570ec5c4c472ad4 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:34:58 +0100 Subject: [PATCH 06/35] Z9 zfs-apply-run: the apply holds the lock, waits on a scrub or an unimported pool, and converges the node's id, ARC and pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §15, §18, A-8, A-16, A-29 --- modules/zfs/cmd/zfs-tools/applied.go | 40 +- modules/zfs/cmd/zfs-tools/applied_test.go | 11 + modules/zfs/cmd/zfs-tools/apply.go | 401 ++++++++++++++++++ modules/zfs/cmd/zfs-tools/apply_devices.go | 11 +- modules/zfs/cmd/zfs-tools/apply_properties.go | 5 +- modules/zfs/cmd/zfs-tools/apply_test.go | 205 +++++++++ modules/zfs/cmd/zfs-tools/bus.go | 48 +++ modules/zfs/cmd/zfs-tools/helpers_test.go | 6 + modules/zfs/cmd/zfs-tools/machine.go | 10 +- modules/zfs/cmd/zfs-tools/main.go | 15 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 8 +- 11 files changed, 739 insertions(+), 21 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/apply.go create mode 100644 modules/zfs/cmd/zfs-tools/apply_test.go create mode 100644 modules/zfs/cmd/zfs-tools/bus.go diff --git a/modules/zfs/cmd/zfs-tools/applied.go b/modules/zfs/cmd/zfs-tools/applied.go index b42fc4537..3d99d978c 100644 --- a/modules/zfs/cmd/zfs-tools/applied.go +++ b/modules/zfs/cmd/zfs-tools/applied.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "fmt" "time" ) @@ -41,6 +42,9 @@ const ( OutcomeRefused = "refused" OutcomeWaiting = "waiting" OutcomeFailed = "failed" + // OutcomeAwaitingLoad is a kernel module parameter set in the options file whose live value differs: it + // takes effect at the next load of the kernel module (§18, §29). + OutcomeAwaitingLoad = "awaiting-load" ) // StateStore is the module's states on the bus, as the process or the tools bundle reaches them. @@ -49,6 +53,8 @@ type StateStore interface { Delete(state, key string) error // Get answers false when the key is not there. Get(state, key string) (json.RawMessage, bool, error) + // Keys are the keys of a state. + Keys(state string) ([]string, error) } // AppliedEntry is one key of the state `applied`. @@ -63,21 +69,47 @@ type AppliedEntry struct { Since time.Time `json:"since,omitzero"` } +// notSaid is a change, refusal or wait the bus did not take: the machine is as the apply left it, but the +// event or the key is missing. It is a warning, never a failed command (§18: with the bus down the apply +// converges from the config file it last got). +type notSaid struct{ err error } + +func (e notSaid) Error() string { return e.err.Error() } +func (e notSaid) Unwrap() error { return e.err } + // say publishes one event of the module. func (m *Machine) say(kind string, body map[string]any) error { if m.Emit == nil { - return fmt.Errorf("this process publishes no events: %s was not said", kind) + return notSaid{fmt.Errorf("this process publishes no events: %s was not said", kind)} } - return m.Emit(kind, body) + if err := m.Emit(kind, body); err != nil { + return notSaid{err} + } + return nil +} + +// failure records one failed step of the apply: a failed command is kept under its key and answered as the +// apply's failure; a change the bus did not take is only a warning. +func (m *Machine) failure(failed *[]error, key string, err error) { + var ns notSaid + if errors.As(err, &ns) { + m.Warnings = append(m.Warnings, fmt.Sprintf("%s: %v", key, err)) + return + } + *failed = append(*failed, fmt.Errorf("%s: %v", key, err)) + _ = m.CommandFailed(key, err) } // keep writes a key of the state `applied`. func (m *Machine) keep(key string, e AppliedEntry) error { if m.States == nil { - return fmt.Errorf("this process keeps no state: %s was not kept", key) + return notSaid{fmt.Errorf("this process keeps no state: %s was not kept", key)} } e.At = m.Now().UTC() - return m.States.Put(StateApplied, key, e) + if err := m.States.Put(StateApplied, key, e); err != nil { + return notSaid{err} + } + return nil } // Changed says a change as `config.applied` {target, key, before, after, read-back} and keeps it. The event diff --git a/modules/zfs/cmd/zfs-tools/applied_test.go b/modules/zfs/cmd/zfs-tools/applied_test.go index d4ac4c462..f0a2bec8d 100644 --- a/modules/zfs/cmd/zfs-tools/applied_test.go +++ b/modules/zfs/cmd/zfs-tools/applied_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "sort" "strings" "testing" "time" @@ -15,6 +16,16 @@ func (s memStates) Put(state, key string, v any) error { return nil } func (s memStates) Delete(state, key string) error { delete(s, state+"/"+key); return nil } +func (s memStates) Keys(state string) ([]string, error) { + var out []string + for k := range s { + if rest, ok := strings.CutPrefix(k, state+"/"); ok { + out = append(out, rest) + } + } + sort.Strings(out) + return out, nil +} func (s memStates) Get(state, key string) (json.RawMessage, bool, error) { b, ok := s[state+"/"+key] return b, ok, nil diff --git a/modules/zfs/cmd/zfs-tools/apply.go b/modules/zfs/cmd/zfs-tools/apply.go new file mode 100644 index 000000000..beabb0ebd --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply.go @@ -0,0 +1,401 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "os" + "regexp" + "strconv" + "strings" + "syscall" +) + +// The apply: who, when, and what it converges (design 42 zfs-owns-zfs §15, §18). +// +// `zfs-tools apply` runs as root, holds an exclusive lock on ApplyLock (a second start waits for it), reads +// the config file the node-engine wrote, and converges the node's ZFS id, the ARC's live parameters, the +// pool and dataset properties and the cache and log devices. It converges nothing while a scrub or resilver +// runs on the pool or before the pool is imported: it says `config.waiting` and ends, and the event follower +// starts it again on the event that ends the wait (§27). It never runs on a timer. It exits 0 on a refusal or +// a wait, and non-zero only when a command it ran failed (A-8), so the node-engine gates the module on a real +// failure. + +// ApplyLock is the lock the apply, the scheduled run and the hand acts take. +const ApplyLock = "/run/zfs-tools/apply.lock" + +// The files the apply reads and writes. +const ( + HostIDFile = "/etc/hostid" + ParametersDir = "/sys/module/zfs/parameters/" + MemInfoFile = "/proc/meminfo" +) + +// ZdbConfigArgv reads a pool's configuration, where its label's hostid is. +func ZdbConfigArgv(pool string) []string { return []string{"-C", pool} } + +// ZgenhostidArgv writes /etc/hostid with ZFS's own writer; the value is one the parse validated (§11). +func ZgenhostidArgv(value string) []string { return []string{"-f", value} } + +// FileLock takes an exclusive lock on a file, waiting for it or answering at once. +func FileLock(path string, wait bool) (func(), bool, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, false, err + } + how := syscall.LOCK_EX + if !wait { + how |= syscall.LOCK_NB + } + for { + err = syscall.Flock(int(f.Fd()), how) + if err != syscall.EINTR { + break + } + } + if err != nil { + f.Close() + if err == syscall.EWOULDBLOCK { + return nil, false, nil + } + return nil, false, err + } + return func() { f.Close() }, true, nil +} + +func writeLive(path string, data []byte) error { return os.WriteFile(path, data, 0o644) } + +// memMiB is the node's memory in MiB from /proc/meminfo, 0 when it cannot be read (A-29). +func (m *Machine) memMiB() uint64 { + b, err := m.ReadFile(MemInfoFile) + if err != nil { + return 0 + } + for _, l := range strings.Split(string(b), "\n") { + if rest, ok := strings.CutPrefix(l, "MemTotal:"); ok { + f := strings.Fields(rest) + if len(f) > 0 { + if kb, err := strconv.ParseUint(f[0], 10, 64); err == nil { + return kb / 1024 + } + } + } + } + return 0 +} + +// Managed says whether the config value a state key belongs to (its words before the first dot) is set. +func (c *Config) Managed(key string) bool { + k, _, _ := strings.Cut(key, ".") + switch k { + case "pool": + return c.Pool != "" + case "scrub-cadence": + return c.ScrubCadence != "" + case "scrub-day": + return c.ScrubDay != "" + case "scrub-time": + return c.ScrubTime != "" + case "arc-max": + return c.ArcMax != nil + case "arc-min": + return c.ArcMin != nil + case "module-options": + return len(c.ModuleOptions) > 0 + case "hostid": + return c.HostID != "" + case "zed": + return c.Zed != "" + case "zed-handlers": + return len(c.ZedHandlers) > 0 + case "trim-schedule": + return c.TrimSchedule != "" + case "trim-day": + return c.TrimDay != "" + case "trim-time": + return c.TrimTime != "" + case "cache-devices": + return len(c.CacheDevices) > 0 + case "log-devices": + return len(c.LogDevices) > 0 + case "log-mirror": + return c.LogMirror != nil + case "pool-properties": + return len(c.PoolProperties) > 0 + case "dataset-properties": + return len(c.DatasetProperties) > 0 + case "snapshots": + return c.Snapshots != nil + } + return false +} + +// scanBusy says what runs on the pool that holds the apply back, "" when nothing does. +func scanBusy(scans []Scan) string { + for _, s := range scans { + if (s.Kind == "scrub" || s.Kind == "resilver") && (s.State == "running" || s.State == "paused") { + return "a " + s.Kind + " is running on the pool" + } + } + return "" +} + +// Apply converges the node to its config file and answers a line of what it did. Its error is a failed +// command, never a refusal or a wait. +func (m *Machine) Apply(ctx context.Context) (string, error) { + release, _, err := m.Lock(ApplyLock, true) + if err != nil { + return "", fmt.Errorf("the apply's lock %s: %v", ApplyLock, err) + } + defer release() + text, err := m.ReadFile(ConfigFile) + if err != nil { + return "", fmt.Errorf("the config file %s cannot be read: %v", ConfigFile, err) + } + c, refusals := ParseConfig(text, m.memMiB()) + if len(refusals) > 0 { + var failed []error + for _, r := range refusals { + if err := m.Refused(r); err != nil { + m.failure(&failed, r.Key, err) + } + } + return fmt.Sprintf("%d values refused; nothing converged", len(refusals)), errors.Join(failed...) + } + m.forget(c) + var failed []error + // The pool must be imported and idle; until then nothing converges. + if c.Pool != "" { + wait, err := m.poolWait(ctx, c.Pool) + if err != nil { + m.failure(&failed, "pool", err) + return "the pool could not be read", errors.Join(failed...) + } + if wait != "" { + if err := m.Waiting("pool", wait); err != nil { + m.failure(&failed, "pool", err) + } + return "waiting for " + wait, errors.Join(failed...) + } + _ = m.NotManaged("pool") + } + if c.HostID != "" && c.Pool == "" { + _ = m.Refused(Refusal{Key: "hostid", Value: c.HostID, Reason: "needs the pool set, to read its label"}) + } else if c.HostID != "" { + m.ApplyHostID(ctx, c, &failed) + } + m.ApplyARC(c, &failed) + if c.Pool != "" { + if err := m.ApplyProperties(ctx, c); err != nil { + failed = append(failed, err) + } + if err := m.ApplyDevices(ctx, c); err != nil { + failed = append(failed, err) + } + } + return "applied", errors.Join(failed...) +} + +// forget deletes the keys of the state `applied` that are a refusal or a wait of a value, or a value now not +// managed: what is still wrong is said again by this apply, and healthy is a key that is not there. +func (m *Machine) forget(c *Config) { + if m.States == nil { + return + } + keys, err := m.States.Keys(StateApplied) + if err != nil { + return + } + for _, k := range keys { + if !c.Managed(k) { + _ = m.States.Delete(StateApplied, k) + continue + } + raw, ok, err := m.States.Get(StateApplied, k) + if err != nil || !ok { + continue + } + if strings.Contains(string(raw), `"outcome":"`+OutcomeRefused+`"`) || strings.Contains(string(raw), `"outcome":"`+OutcomeFailed+`"`) { + _ = m.States.Delete(StateApplied, k) + } + } +} + +// poolWait is what the apply waits on, "" when the pool is imported and idle. +func (m *Machine) poolWait(ctx context.Context, pool string) (string, error) { + pools, err := m.Pools(ctx) + if err != nil { + return "", err + } + found := false + for _, p := range pools { + found = found || p.Name == pool + } + if !found { + return "the pool " + pool + " to be imported", nil + } + out, err := m.ask(ctx, "zpool", StatusArgv()...) + if err != nil { + return "", err + } + ps, err := ParseStatus(out) + if err != nil { + return "", err + } + for _, p := range ps { + if p.Name == pool { + if busy := scanBusy(p.Scans); busy != "" { + return "the scrub or resilver of the pool " + pool + " to end (" + busy + ")", nil + } + } + } + return "", nil +} + +var labelHostID = regexp.MustCompile(`(?m)^\s*hostid:\s*(\d+)`) + +func nativeHostID(b []byte) (uint32, bool) { + if len(b) != 4 { + return 0, false + } + return binary.NativeEndian.Uint32(b), true +} + +// ApplyHostID converges /etc/hostid to the set value, only when the pool's label is 0 or already that value +// (§15), and reads the file back. +func (m *Machine) ApplyHostID(ctx context.Context, c *Config, failed *[]error) { + const key = "hostid" + refuse := func(why string) { + if err := m.Refused(Refusal{Key: key, Value: c.HostID, Reason: why}); err != nil { + m.failure(failed, key, err) + } + } + n, _ := strconv.ParseUint(c.HostID, 16, 32) + want := uint32(n) + current := func() string { + b, err := m.ReadFile(HostIDFile) + if err != nil { + return "not there" + } + if v, ok := nativeHostID(b); ok { + return fmt.Sprintf("%08x", v) + } + return "not four bytes" + } + r, err := m.AsRoot(ctx, "zdb", ZdbConfigArgv(c.Pool)...) + if err != nil { + refuse("the pool's label cannot be read: " + err.Error()) + return + } + lm := labelHostID.FindStringSubmatch(r.Out) + if lm == nil { + refuse("the pool's label cannot be read: zdb -C shows no hostid") + return + } + label, _ := strconv.ParseUint(lm[1], 10, 32) + if label != 0 && uint32(label) != want { + refuse(fmt.Sprintf("the pool's import would need -f: its label holds the hostid %08x, not %s", label, c.HostID)) + return + } + before := current() + if before != c.HostID { + if _, err := m.AsRoot(ctx, "zgenhostid", ZgenhostidArgv(c.HostID)...); err != nil { + m.failure(failed, key, err) + return + } + } + back := current() + if back != c.HostID { + refuse(fmt.Sprintf("%s holds %s after the write, not the set %s: another writer changed it", HostIDFile, back, c.HostID)) + return + } + if before == back { + if err := m.keepAsSet(key, back); err != nil { + m.failure(failed, key, err) + } + return + } + if err := m.Changed("file", key, before, c.HostID, back); err != nil { + m.failure(failed, key, err) + } +} + +// ApplyARC writes arc-max and arc-min live and reads both back; a module option whose live value differs is +// a warning that it takes effect at the next load of the kernel module, never a failure. +func (m *Machine) ApplyARC(c *Config, failed *[]error) { + params := []struct { + key, file string + mib *uint64 + }{{"arc-max", "zfs_arc_max", c.ArcMax}, {"arc-min", "zfs_arc_min", c.ArcMin}} + live := func(file string) (string, error) { + b, err := m.ReadFile(ParametersDir + file) + return strings.TrimSpace(string(b)), err + } + before := map[string]string{} + for _, p := range params { + if p.mib == nil { + continue + } + v, err := live(p.file) + if err != nil { + m.failure(failed, p.key, fmt.Errorf("%s cannot be read: %v", ParametersDir+p.file, err)) + continue + } + before[p.key] = v + } + // arc-min may not pass arc-max while ZFS holds them, so a second pass settles an order that failed. + for pass := 0; pass < 2; pass++ { + for _, p := range params { + if p.mib == nil || before[p.key] == "" { + continue + } + want := strconv.FormatUint(*p.mib<<20, 10) + if v, _ := live(p.file); v != want { + if err := m.WriteFile(ParametersDir+p.file, []byte(want)); err != nil && pass == 1 { + m.failure(failed, p.key, err) + } + } + } + } + for _, p := range params { + if p.mib == nil || before[p.key] == "" { + continue + } + want := strconv.FormatUint(*p.mib<<20, 10) + back, _ := live(p.file) + var err error + switch { + case back != want: + err = fmt.Errorf("%s reads %q after writing %q", ParametersDir+p.file, back, want) + m.failure(failed, p.key, err) + case before[p.key] == want: + err = m.keepAsSet(p.key, back) + default: + err = m.Changed("file", p.key, before[p.key], want, back) + } + if err != nil && back == want { + m.failure(failed, p.key, err) + } + } + for _, o := range c.ModuleOptions { + key := "module-options." + o.Name + v, err := live(o.Name) + switch { + case err == nil && v == o.Value: + if err := m.keepAsSet(key, v); err != nil { + m.failure(failed, key, err) + } + default: + if err == nil { + m.Warnings = append(m.Warnings, fmt.Sprintf("%s: live %s, set %s: takes effect at the next load of the kernel module", o.Name, v, o.Value)) + } else { + m.Warnings = append(m.Warnings, fmt.Sprintf("%s: not a live parameter here: takes effect at the next load of the kernel module", o.Name)) + } + if err := m.keep(key, AppliedEntry{Before: v, After: o.Value, Outcome: OutcomeAwaitingLoad, + Why: "takes effect at the next load of the kernel module"}); err != nil { + m.failure(failed, key, err) + } + } + } +} diff --git a/modules/zfs/cmd/zfs-tools/apply_devices.go b/modules/zfs/cmd/zfs-tools/apply_devices.go index 191d0d2df..961629ca6 100644 --- a/modules/zfs/cmd/zfs-tools/apply_devices.go +++ b/modules/zfs/cmd/zfs-tools/apply_devices.go @@ -145,10 +145,8 @@ func (m *Machine) devicesOf(ctx context.Context, pool string) (*poolDevices, err case real.Root != nil && deviceErrors(real.Root) > 0: pd.Healthy, pd.Why = false, "a device of the pool has read, write or checksum errors" } - for _, s := range real.Scans { - if (s.Kind == "scrub" || s.Kind == "resilver") && (s.State == "running" || s.State == "paused") { - pd.Healthy, pd.Why = false, "a "+s.Kind+" is running on the pool" - } + if busy := scanBusy(real.Scans); busy != "" { + pd.Healthy, pd.Why = false, busy } for _, role := range []string{"cache", "log"} { hs, rs := sectionOf(held, role), sectionOf(real, role) @@ -302,10 +300,7 @@ func (m *Machine) ApplyDevices(ctx context.Context, c *Config) error { return nil } var failed []error - fail := func(key string, err error) { - failed = append(failed, fmt.Errorf("%s: %v", key, err)) - _ = m.CommandFailed(key, err) - } + fail := func(key string, err error) { m.failure(&failed, key, err) } others, err := m.otherPoolDevices(ctx, c.Pool) if err != nil { fail("cache-devices", err) diff --git a/modules/zfs/cmd/zfs-tools/apply_properties.go b/modules/zfs/cmd/zfs-tools/apply_properties.go index 495bf4ffa..3c287ce82 100644 --- a/modules/zfs/cmd/zfs-tools/apply_properties.go +++ b/modules/zfs/cmd/zfs-tools/apply_properties.go @@ -75,10 +75,7 @@ func sameValue(property, got, want string) bool { // one error; a property whose read-back differs from what was set is a failure too. func (m *Machine) ApplyProperties(ctx context.Context, c *Config) error { var failed []error - fail := func(key string, err error) { - failed = append(failed, fmt.Errorf("%s: %v", key, err)) - _ = m.CommandFailed(key, err) - } + fail := func(key string, err error) { m.failure(&failed, key, err) } for _, p := range sortedKeys(c.PoolProperties) { key := "pool-properties." + p want := c.PoolProperties[p] diff --git a/modules/zfs/cmd/zfs-tools/apply_test.go b/modules/zfs/cmd/zfs-tools/apply_test.go new file mode 100644 index 000000000..fd5eac58e --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/apply_test.go @@ -0,0 +1,205 @@ +package main + +import ( + "context" + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" +) + +func hostidFile(v uint32) string { + b := make([]byte, 4) + binary.NativeEndian.PutUint32(b, v) + return string(b) +} + +// applyFake is a machine whose pool tank is imported and idle, with the memory and the ARC files of a node. +func applyFake(t *testing.T, conf string) *fake { + f := newFake(t) + f.uid = 0 + f.files[ConfigFile] = conf + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list.txt")) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status.txt")) + f.files[ParametersDir+"zfs_arc_max"] = "0\n" + f.files[ParametersDir+"zfs_arc_min"] = "0\n" + return f +} + +func hostidCase(f *fake, label, file string) { + f.on("zdb -C tank", "MOS Configuration:\n version: 5000\n name: 'tank'\n hostid: "+label+"\n") + if file != "" { + f.files[HostIDFile] = file + } +} + +// §15, §31: unset: no zgenhostid; a label of another id or one that cannot be read: refused; 0 or equal: one +// run; a file holding another value after the write: refused; each refusal unhealthy and healthy again once +// the label and the file agree or hostid is unset. +func TestTheHostIDIsConvergedOnlyWhereTheLabelAllowsIt(t *testing.T) { + const want = "00bab10c" // 12235020 + cases := []struct { + name, conf, label, file string + zdbErr bool + writes bool + event string + outcome string + }{ + {name: "unset", conf: `{"pool":"tank"}`}, + {name: "label of another id", conf: `{"pool":"tank","hostid":"` + want + `"}`, label: "12345", event: "config.refused", outcome: "refused"}, + {name: "label cannot be read", conf: `{"pool":"tank","hostid":"` + want + `"}`, zdbErr: true, event: "config.refused", outcome: "refused"}, + {name: "label 0", conf: `{"pool":"tank","hostid":"` + want + `"}`, label: "0", writes: true, event: "config.applied", outcome: "applied"}, + {name: "label equal", conf: `{"pool":"tank","hostid":"` + want + `"}`, label: "12235020", writes: true, event: "config.applied", outcome: "applied"}, + {name: "file already equal", conf: `{"pool":"tank","hostid":"` + want + `"}`, label: "12235020", file: hostidFile(12235020), outcome: "applied"}, + {name: "another writer's value after the write", conf: `{"pool":"tank","hostid":"` + want + `"}`, label: "0", file: hostidFile(7), writes: true, event: "config.refused", outcome: "refused"}, + } + for _, c := range cases { + f := applyFake(t, c.conf) + if c.zdbErr { + f.answers["zdb -C tank"] = Ran{Code: 1, Err: "cannot open"} + } else if c.label != "" { + hostidCase(f, c.label, c.file) + } + if c.writes { + f.on("zgenhostid -f "+want, "") + f.after["zgenhostid -f "+want] = func(f *fake) { + if c.name != "another writer's value after the write" { + f.files[HostIDFile] = hostidFile(12235020) + } + } + } + m := f.machine() + st := memStates{} + m.States = st + if _, err := m.Apply(context.Background()); err != nil { + t.Errorf("%s: %v", c.name, err) + } + if n := len(ran(f, "zgenhostid")); (n == 1) != c.writes || n > 1 { + t.Errorf("%s: zgenhostid ran %d times", c.name, n) + } + if c.event == "" && len(f.emitted) != 0 || c.event != "" && (len(f.emitted) != 1 || f.emitted[0]["_type"] != c.event) { + t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event) + } + if raw, ok := st["applied/hostid"]; c.outcome != "" && (!ok || !strings.Contains(string(raw), `"outcome":"`+c.outcome+`"`)) { + t.Errorf("%s: state %v", c.name, st) + } + if c.outcome == "refused" { + // Healthy again once hostid is unset. + f.files[ConfigFile] = `{"pool":"tank"}` + if _, err := m.Apply(context.Background()); err != nil { + t.Error(err) + } + if _, ok := st["applied/hostid"]; ok { + t.Errorf("%s: the refusal stayed after hostid was unset", c.name) + } + } + } +} + +// §18: the lock is waited for; a scrub, resilver or unimported pool is a wait and the apply ends with exit 0; +// exit non-zero only for a failed command; ARC values are written live and read back, a differing module +// option is a warning; with the bus down it converges from the config file. +func TestTheApplyWaitsConvergesAndExitsAsADR0679Says(t *testing.T) { + t.Run("lock waited for, ARC written and read back", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","arc-max":4096,"arc-min":512}`) + m := f.machine() + m.States = memStates{} + if _, err := m.Apply(context.Background()); err != nil { + t.Fatal(err) + } + if len(f.locks) != 1 || !f.locks[0] { + t.Errorf("locks %v: the apply must wait for the lock", f.locks) + } + if f.files[ParametersDir+"zfs_arc_max"] != "4294967296" || f.files[ParametersDir+"zfs_arc_min"] != "536870912" { + t.Errorf("ARC %q %q", f.files[ParametersDir+"zfs_arc_max"], f.files[ParametersDir+"zfs_arc_min"]) + } + if len(f.emitted) != 2 || f.emitted[0]["read-back"] != "4294967296" { + t.Errorf("events %v", f.emitted) + } + }) + t.Run("a scrub running is a wait", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","arc-max":4096}`) + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt")) + m := f.machine() + m.States = memStates{} + if _, err := m.Apply(context.Background()); err != nil { + t.Fatal(err) + } + if len(f.emitted) != 1 || f.emitted[0]["_type"] != "config.waiting" || f.files[ParametersDir+"zfs_arc_max"] != "0\n" { + t.Errorf("events %v", f.emitted) + } + }) + t.Run("an unimported pool is a wait", func(t *testing.T) { + f := applyFake(t, `{"pool":"nopool","arc-max":4096}`) + m := f.machine() + m.States = memStates{} + if _, err := m.Apply(context.Background()); err != nil || len(f.emitted) != 1 || f.emitted[0]["_type"] != "config.waiting" { + t.Errorf("%v %v", err, f.emitted) + } + }) + t.Run("a refused value refuses the whole file, exit 0", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","arc-max":4096,"module-options":["bogus=1"]}`) + m := f.machine() + m.States = memStates{} + if _, err := m.Apply(context.Background()); err != nil || len(f.emitted) != 1 || f.emitted[0]["_type"] != "config.refused" || + f.files[ParametersDir+"zfs_arc_max"] != "0\n" { + t.Errorf("%v %v", err, f.emitted) + } + }) + t.Run("a failed command is the only non-zero exit", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","pool-properties":{"autotrim":"on"}}`) + f.on("zpool get -Hp -o value autotrim -- tank", "off\n") + f.answers["zpool set autotrim=on -- tank"] = Ran{Code: 1, Err: "cannot set"} + m := f.machine() + m.States = memStates{} + if _, err := m.Apply(context.Background()); err == nil { + t.Error("a failed zpool set exited 0") + } + }) + t.Run("a differing module option is a warning, never a failure", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","module-options":["zfs_txg_timeout=10"]}`) + f.files[ParametersDir+"zfs_txg_timeout"] = "5\n" + m := f.machine() + st := memStates{} + m.States = st + if _, err := m.Apply(context.Background()); err != nil { + t.Fatal(err) + } + if len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "next load of the kernel module") || + !strings.Contains(string(st["applied/module-options.zfs_txg_timeout"]), OutcomeAwaitingLoad) || len(f.emitted) != 0 { + t.Errorf("%v %v %v", m.Warnings, st, f.emitted) + } + }) + t.Run("with the bus down it converges from the config file", func(t *testing.T) { + f := applyFake(t, `{"pool":"tank","arc-max":4096}`) + m := f.machine() + m.Emit, m.States = nil, nil + if _, err := m.Apply(context.Background()); err != nil { + t.Fatal(err) + } + if f.files[ParametersDir+"zfs_arc_max"] != "4294967296" || len(m.Warnings) == 0 { + t.Errorf("ARC %q, warnings %v", f.files[ParametersDir+"zfs_arc_max"], m.Warnings) + } + }) +} + +// §18: FileLock holds a second taker out, and a second one that waits gets it once the first lets go. +func TestFileLockHoldsASecondTakerOut(t *testing.T) { + path := filepath.Join(t.TempDir(), "apply.lock") + release, got, err := FileLock(path, false) + if err != nil || !got { + t.Fatal(err, got) + } + if _, got2, err := FileLock(path, false); err != nil || got2 { + t.Errorf("second taker: %v %v", got2, err) + } + release() + rel2, got3, err := FileLock(path, false) + if err != nil || !got3 { + t.Errorf("after release: %v %v", got3, err) + } + rel2() + _ = os.Remove(path) +} diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go new file mode 100644 index 000000000..8737c117e --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/json" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// busStates is the module's states as this process reaches them (its bus account's `state`). +type busStates struct{ c *process.Conn } + +func (b busStates) Put(state, key string, v any) error { + _, err := b.c.State(state).Put(key, v) + return err +} +func (b busStates) Delete(state, key string) error { return b.c.State(state).Delete(key) } +func (b busStates) Keys(state string) ([]string, error) { return b.c.State(state).Keys() } +func (b busStates) Get(state, key string) (json.RawMessage, bool, error) { + e, err := b.c.State(state).Get(key) + if err != nil { + return nil, false, err + } + if e == nil { + return nil, false, nil + } + return e.Value, true, nil +} + +// JoinBus gives the machine this process's events and states, as its own bus account says. Without the +// account or the bus the machine keeps none, and what it changes is said as a warning (§18: with the bus down +// the apply converges from the config file it last got). It answers how to leave the bus. +func (m *Machine) JoinBus() func() { + acct, err := process.ReadBrokerAccount(os.Getenv) + if err != nil { + m.Warnings = append(m.Warnings, err.Error()) + return func() {} + } + c, err := process.Connect(acct) + if err != nil { + m.Warnings = append(m.Warnings, err.Error()) + return func() {} + } + m.Emit = func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) } + m.States = busStates{c} + return c.Close +} diff --git a/modules/zfs/cmd/zfs-tools/helpers_test.go b/modules/zfs/cmd/zfs-tools/helpers_test.go index 9a245c587..a8707d333 100644 --- a/modules/zfs/cmd/zfs-tools/helpers_test.go +++ b/modules/zfs/cmd/zfs-tools/helpers_test.go @@ -33,6 +33,7 @@ type fake struct { emitErr error files map[string]string links map[string]string + locks []bool // the wait of each lock taken now time.Time uid int errors *ErrorMemory @@ -81,6 +82,11 @@ func (f *fake) machine() *Machine { } return "", os.ErrNotExist }, + WriteFile: func(p string, b []byte) error { f.files[p] = string(b); return nil }, + Lock: func(path string, wait bool) (func(), bool, error) { + f.locks = append(f.locks, wait) + return func() {}, true, nil + }, Errors: f.errors, Emit: func(eventType string, body any) error { if f.emitErr != nil { diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 3164bfacc..49ed8f0aa 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -112,6 +112,14 @@ type Machine struct { Now func() time.Time // ReadFile reads a kernel file (the ARC's counters). ReadFile func(path string) ([]byte, error) + // WriteFile writes a file as root (the ARC's live parameters). + WriteFile func(path string, data []byte) error + // Lock takes an exclusive lock on a file: waiting for it, or answering at once whether it was got. The + // release gives it up. + Lock func(path string, wait bool) (release func(), got bool, err error) + // Warnings are what the apply saw that is no failure (a change the bus did not take, a parameter awaiting + // a load). + Warnings []string // Resolve follows a link to its kernel path (a declared device's name under /dev/disk/by-id). Resolve func(path string) (string, error) // Emit publishes an event as this module (the record of an act); nil where nothing can be published. @@ -125,7 +133,7 @@ type Machine struct { // Here is the machine this process runs on. func Here() *Machine { - return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Errors: LoadErrorMemory(ErrorsFile)} + return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, WriteFile: writeLive, Lock: FileLock, Errors: LoadErrorMemory(ErrorsFile)} } // NotApplicable is the answer of a machine without ZFS: the module does not apply here. diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 4d3c95356..18d079383 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -34,6 +34,19 @@ func main() { os.Exit(1) } fmt.Println(said) + case "apply": + m := Here() + closeBus := m.JoinBus() + said, err := m.Apply(ctx) + closeBus() + for _, w := range m.Warnings { + fmt.Fprintln(os.Stderr, "warning:", w) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(said) case "events": if err := Follower(ctx, os.Stdout); err != nil { fmt.Fprintln(os.Stderr, err) @@ -41,7 +54,7 @@ func main() { } default: fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+ - "watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub\n", os.Args[1]) + "watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub, with apply it converges the node to its config file\n", os.Args[1]) os.Exit(2) } return diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index eca905d50..93ca39be8 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -222,6 +222,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, "DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"}, "AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"}, + "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), @@ -232,10 +233,11 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DatasetInheritArgv": DatasetInheritArgv("tank", "atime"), "DeviceStatusArgv": DeviceStatusArgv("tank", true), "AllRealStatusArgv": AllRealStatusArgv(), "RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"), - "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x")} + "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x"), + "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")} for fn, argv := range verbs { for _, a := range argv { - if fn != "FollowArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") { + if fn != "FollowArgv" && fn != "ZgenhostidArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") { t.Errorf("%s runs %q", fn, a) } } @@ -253,7 +255,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { } } call := regexp.MustCompile(`(\.(ask|AsRoot|Escalated|Run|Start|Output|CombinedOutput)\(|exec\.Command(Context)?\(|os\.StartProcess\(|syscall\.(Exec|ForkExec)\()`) - site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb)", (\w+)\(`) + site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb|zgenhostid)", (\w+)\(`) // The bodies that run what they are given, each once. bodies := map[string]bool{ "machine.go: r := m.Run(ctx, name, args...)": true, From 06d83912cdbdae40e8553368899164203da6f678 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:38:17 +0100 Subject: [PATCH 07/35] Z10 zfs-run: the scheduled run takes the lock, starts a due scrub and trim without waiting, and acts on hand acts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §20, §24, A-17 --- modules/zfs/cmd/zfs-tools/apply.go | 5 + modules/zfs/cmd/zfs-tools/main.go | 16 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 26 +- modules/zfs/cmd/zfs-tools/run.go | 168 +++++++++++ modules/zfs/cmd/zfs-tools/run_test.go | 269 ++++++++++++++++++ modules/zfs/cmd/zfs-tools/schedule.go | 147 ++-------- modules/zfs/cmd/zfs-tools/scrub.go | 2 +- modules/zfs/cmd/zfs-tools/scrubrun_test.go | 206 -------------- .../zfs/cmd/zfs-tools/scrubschedule_test.go | 119 -------- modules/zfs/module.json | 6 +- 10 files changed, 502 insertions(+), 462 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/run.go create mode 100644 modules/zfs/cmd/zfs-tools/run_test.go delete mode 100644 modules/zfs/cmd/zfs-tools/scrubrun_test.go delete mode 100644 modules/zfs/cmd/zfs-tools/scrubschedule_test.go diff --git a/modules/zfs/cmd/zfs-tools/apply.go b/modules/zfs/cmd/zfs-tools/apply.go index beabb0ebd..bcd5ed579 100644 --- a/modules/zfs/cmd/zfs-tools/apply.go +++ b/modules/zfs/cmd/zfs-tools/apply.go @@ -149,6 +149,11 @@ func (m *Machine) Apply(ctx context.Context) (string, error) { return "", fmt.Errorf("the apply's lock %s: %v", ApplyLock, err) } defer release() + return m.applyLocked(ctx) +} + +// applyLocked is the apply under a lock already held (the scheduled run's hand act takes the same lock). +func (m *Machine) applyLocked(ctx context.Context) (string, error) { text, err := m.ReadFile(ConfigFile) if err != nil { return "", fmt.Errorf("the config file %s cannot be read: %v", ConfigFile, err) diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 18d079383..1056e65e3 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -6,8 +6,8 @@ // speaks MCP to over stdio (novox/hq ADR 0188, ADR 0193), as the operator account. With `watch` and // `events` it is one of the module's two long-running processes, run by the node-engine: the watch, which // the health check zfs_health stands on (watch.go), and the event follower, as root, judged by its unit -// (follow.go). With `scrub-run` it is the scheduled scrub, run as root by the node-engine's timer -// (schedule.go). +// (follow.go). With `run` it is the scheduled run, as root by the node-engine's timer (run.go), and with `apply` the +// apply that converges the node to its config file (apply.go). package main import ( @@ -27,8 +27,14 @@ func main() { switch os.Args[1] { case "watch": (&Watch{M: Here(), Out: os.Stdout}).Loop(ctx) - case "scrub-run": - said, err := Here().ScrubRun(ctx, ServedFile) + case "run": + m := Here() + closeBus := m.JoinBus() + said, err := m.RunScheduled(ctx, ServedFile) + closeBus() + for _, w := range m.Warnings { + fmt.Fprintln(os.Stderr, "warning:", w) + } if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -54,7 +60,7 @@ func main() { } default: fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+ - "watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub, with apply it converges the node to its config file\n", os.Args[1]) + "watches the pools, with events it follows ZFS's events, with run it runs the scheduled scrub, trim and snapshots, with apply it converges the node to its config file\n", os.Args[1]) os.Exit(2) } return diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 93ca39be8..079bbc266 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -119,10 +119,10 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { byID[r["id"].(string)] = r order[r["id"].(string)] = i } - run := byID["scrub-run"] + run := byID["run"] args, _ := run["run"].([]any) - if run["type"] != "process" || run["name"] != "zfs-mesh-scrub" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || - len(args) != 2 || args[1] != "scrub-run" { + if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || + len(args) != 2 || args[1] != "run" { t.Errorf("the scheduled run: %v", run) } // The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never @@ -131,15 +131,15 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { if s, _ := run["schedule"].(string); strings.Contains(s, "/") { t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s) } - if ScrubTimer != run["name"].(string)+".timer" || ScrubService != run["name"].(string)+".service" { - t.Errorf("the tools read %s and %s; the node-engine names them after %v", ScrubTimer, ScrubService, run["name"]) + if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" { + t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"]) } for _, c := range []string{"weekly", "monthly"} { d := byID["distribution-"+c] if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" { t.Errorf("the distribution's %s timer: %v", c, d) } - if order["scrub-run"] > order["distribution-"+c] { + if order["run"] > order["distribution-"+c] { t.Errorf("the distribution's %s timer is stopped before the run is installed", c) } } @@ -188,7 +188,7 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { conf = fill.Replace(templateText(t, r)) } } - want := []string{"zfs-mesh-scrub.timer running/enabled (the node-engine's, every quarter of an hour)", + want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)", "zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"} if strings.Join(units, "\n") != strings.Join(want, "\n") { t.Errorf("units:\n%s", strings.Join(units, "\n")) @@ -196,9 +196,11 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { for _, u := range units { t.Log(u) } - s, err := ParseScrubConfig(conf) - if err != nil || s.Pool != "storage" || s.Cadence != "monthly" { - t.Fatalf("configuration %q: %v", conf, err) + _ = conf + cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000) + s := ScheduleOf(cfg) + if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" { + t.Fatalf("configuration: %v", refused) } t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04")) if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) { @@ -216,7 +218,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "SnapshotArgv": {"zfs", "snapshot"}, "PoolsArgv": {"zpool", "list"}, "StatusArgv": {"zpool", "status"}, "IOArgv": {"zpool", "iostat"}, "EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"}, - "ScrubArgv": {"zpool", "scrub"}, "WaitArgv": {"zpool", "wait"}, + "ScrubArgv": {"zpool", "scrub"}, "TrimArgv": {"zpool", "trim"}, "TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"}, "PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"}, "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, @@ -227,7 +229,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), "PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(), - "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"), + "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "TrimArgv": TrimArgv("tank"), "PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"), "DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"), "DatasetInheritArgv": DatasetInheritArgv("tank", "atime"), diff --git a/modules/zfs/cmd/zfs-tools/run.go b/modules/zfs/cmd/zfs-tools/run.go new file mode 100644 index 000000000..922cefdf9 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/run.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" +) + +// The scheduled run (design 42 zfs-owns-zfs §20, §24). +// +// `zfs-tools run` is the module's one scheduled run, root's, fired by the node-engine's timer every quarter of +// an hour. Each run takes the apply's lock; held, it does nothing and says `run.skipped` {reason: lock-held}. +// It does nothing while a scrub or resilver runs or before the pool is imported, saying that reason. +// Otherwise it acts on the hand acts waiting in the state `schedule`, starts the scrub when due and returns +// at once (the scrub's end is the event follower's to hear, §27), and starts a trim when trim-schedule is due. + +// The reasons of `run.skipped`. +const ( + SkipLockHeld = "lock-held" + SkipPoolNotImported = "pool-not-imported" + SkipScanRunning = "scrub-or-resilver-running" +) + +// KeyPaused is the key of the state `schedule` that holds the schedule's snapshots (§24). +const KeyPaused = "paused" + +// TrimArgv starts a trim of the pool. +func TrimArgv(pool string) []string { return []string{"trim", "--", pool} } + +// handActs are the acts waiting in the state `schedule`, each under its own key (A-17), but the pause. +func (m *Machine) handActs() []string { + if m.States == nil { + return nil + } + keys, err := m.States.Keys(StateSchedule) + if err != nil { + return nil + } + var acts []string + for _, k := range keys { + if k != KeyPaused { + acts = append(acts, k) + } + } + sort.Strings(acts) + return acts +} + +func (m *Machine) skip(reason string) string { + var failed []error + if err := m.say(EventRunSkipped, map[string]any{"reason": reason}); err != nil { + m.failure(&failed, "run.skipped", err) + } + return "skipped: " + reason +} + +// RunScheduled is the scheduled run. It answers what it did in a line; its error is a command that failed. +func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, error) { + release, got, err := m.Lock(ApplyLock, false) + if err != nil { + return "", fmt.Errorf("the lock %s: %v", ApplyLock, err) + } + if !got { + return m.skip(SkipLockHeld), nil + } + defer release() + text, err := m.ReadFile(ConfigFile) + if err != nil { + return "", fmt.Errorf("%s could not be read: %v", ConfigFile, err) + } + c, refused := ParseConfig(text, m.memMiB()) + if len(refused) > 0 { + return "refused, nothing run: " + refused[0].Key + " " + refused[0].Reason, nil + } + var said []string + var failed []error + acts := m.handActs() + guard := "" + var p *PoolStatus + if c.Pool != "" { + wait, perr := m.poolWait(ctx, c.Pool) + switch { + case perr != nil: + return "", perr + case strings.Contains(wait, "to be imported"): + guard = SkipPoolNotImported + case wait != "": + guard = SkipScanRunning + } + if guard == "" { + st, err := m.Status(ctx) + if err != nil { + return "", err + } + p = statusOf(st, c.Pool) + } + } + for _, act := range acts { + switch { + case act == "apply": + s, err := m.applyLocked(ctx) + said = append(said, "apply: "+s) + if err != nil { + failed = append(failed, err) + } + case guard != "": + said = append(said, m.skip(guard)) + default: + m.Warnings = append(m.Warnings, "the hand act "+act+" is not one this run knows") + } + _ = m.States.Delete(StateSchedule, act) + } + if guard != "" || c.Pool == "" { + if guard != "" && len(acts) == 0 { + said = append(said, m.skip(guard)) + } + return strings.Join(said, "; "), joinErrors(failed) + } + served := m.readServed(servedPath) + now := m.Now() + for _, work := range []struct { + name, key, verb string + s *Schedule + start func() error + }{{"scrub", c.Pool, "start", ScheduleOf(c), func() error { + _, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", c.Pool)...) + return err + }}, {"trim", c.Pool + "/trim", "start", TrimScheduleOf(c), func() error { + _, err := m.AsRoot(ctx, "zpool", TrimArgv(c.Pool)...) + return err + }}} { + if work.s == nil { + continue + } + slot := work.s.Slot(now) + if slot.IsZero() || served[work.key] >= slot.Unix() { + continue + } + if work.name == "scrub" && startedSince(p, slot) { + said = append(said, fmt.Sprintf("the scrub of %s scheduled %s ran already", c.Pool, slot.Format("2006-01-02 15:04"))) + } else if err := work.start(); err != nil && !strings.Contains(err.Error(), "currently") { + failed = append(failed, fmt.Errorf("the scheduled %s of %s could not start: %v", work.name, c.Pool, err)) + continue + } else { + said = append(said, fmt.Sprintf("started the %s of %s scheduled %s", work.name, c.Pool, slot.Format("2006-01-02 15:04"))) + } + served[work.key] = slot.Unix() + if err := writeServed(servedPath, served); err != nil { + failed = append(failed, fmt.Errorf("the %s of %s was started, and the served slot could not be kept: %v", work.name, c.Pool, err)) + } + } + if len(said) == 0 { + said = append(said, "nothing due") + } + return strings.Join(said, "; "), joinErrors(failed) +} + +func joinErrors(errs []error) error { + if len(errs) == 0 { + return nil + } + var parts []string + for _, e := range errs { + parts = append(parts, e.Error()) + } + return fmt.Errorf("%s", strings.Join(parts, "; ")) +} diff --git a/modules/zfs/cmd/zfs-tools/run_test.go b/modules/zfs/cmd/zfs-tools/run_test.go new file mode 100644 index 000000000..2be13467b --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/run_test.go @@ -0,0 +1,269 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func scrubConf(pool, cadence, day, at string) string { + return `{"pool":"` + pool + `","scrub-cadence":"` + cadence + `","scrub-day":"` + day + `","scrub-time":"` + at + `"}` +} + +func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) { + loc := time.Local + at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) } + cases := []struct { + conf string + now time.Time + want time.Time + }{ + {scrubConf("tank", "weekly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, + {scrubConf("tank", "weekly", "Fri", "13:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)}, + {scrubConf("tank", "weekly", "Fri", "11:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)}, + {scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, + {scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)}, + {scrubConf("tank", "monthly", "Thu", "06:00"), at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)}, + } + for _, c := range cases { + cfg, ref := ParseConfig([]byte(c.conf), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + s := ScheduleOf(cfg) + if got := s.Slot(c.now); !got.Equal(c.want) { + t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want) + } + if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) { + t.Errorf("%q at %s: next slot %s", c.conf, c.now, next) + } + } + // A schedule with a value unset is no schedule; a value of another form is refused. + if cfg, _ := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly"}`), 16000); ScheduleOf(cfg) != nil { + t.Error("a schedule without day and time") + } + for _, conf := range []string{scrubConf("tank", "daily", "Mon", "00:00"), scrubConf("tank", "weekly", "Monday", "00:00"), + scrubConf("tank", "weekly", "Mon", "25:00"), scrubConf("tank x", "weekly", "Mon", "00:00"), scrubConf("tank", "weekly", "Mon..Fri", "00:00"), + scrubConf("tank", "weekly", "Mon", "*:*")} { + if _, ref := ParseConfig([]byte(conf), 16000); len(ref) == 0 { + t.Errorf("accepted %q", conf) + } + } +} + +// runFake is a root machine on the healthy fixtures, with the config file, the schedule state and the lock. +func runFake(t *testing.T, conf string) (*fake, *Machine, memStates, string) { + f := withZFS(t) + f.uid = 0 + f.files[ConfigFile] = conf + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + m := f.machine() + st := memStates{} + m.States = st + return f, m, st, filepath.Join(t.TempDir(), "zfs-scrub", "scrub-served.json") +} + +// §20, §24, §31: the lock held: nothing run; a scrub or resilver running or an unimported pool: nothing run +// and said; a due scrub started and the run ends without waiting; a due trim by trim-day and trim-time; a +// served slot not run again; a hand act's key acted on under the lock and deleted. +func TestTheScheduledRunDoesWhatIsDueAndEndsWithoutWaiting(t *testing.T) { + const trim = `{"pool":"tank","trim-schedule":"weekly","trim-day":"Mon","trim-time":"00:00"}` + cases := []struct { + name, conf string + setup func(f *fake, st memStates) + ran, not []string + event string // event kind and reason + served bool + said string + wantErr bool + }{ + {name: "scrub due", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { f.on("zpool scrub -- tank", "") }, + ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait"}, served: true, said: "started the scrub"}, + {name: "scrub refused as one just began", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing"} + }, ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait", "scrub -p", "scrub -s"}, served: true}, + {name: "scrub running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt")) + }, not: []string{"zpool scrub", "zpool wait"}, event: "run.skipped/" + SkipScanRunning}, + {name: "resilver running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + }, not: []string{"zpool scrub"}, event: "run.skipped/" + SkipScanRunning}, + {name: "pool not imported", conf: scrubConf("nopool", "monthly", "Mon", "00:00"), not: []string{"zpool scrub"}, + event: "run.skipped/" + SkipPoolNotImported}, + {name: "slot already served by a scrub that started in it", conf: scrubConf("tank", "weekly", "Sun", "00:00"), setup: func(f *fake, _ memStates) { + f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local) + }, not: []string{"zpool scrub"}, served: true, said: "ran already"}, + {name: "trim due", conf: trim, setup: func(f *fake, _ memStates) { f.on("zpool trim -- tank", "") }, + ran: []string{"zpool trim -- tank"}, served: true, said: "started the trim"}, + {name: "trim none", conf: `{"pool":"tank","trim-schedule":"none","trim-day":"Mon","trim-time":"00:00"}`, not: []string{"zpool trim"}}, + {name: "a hand act's key is acted on and deleted", conf: `{"pool":"tank"}`, setup: func(f *fake, st memStates) { + st.Put(StateSchedule, "apply", map[string]string{"by": "x"}) + }, said: "apply: applied"}, + {name: "a refused config runs nothing", conf: `{"pool":"tank","scrub-cadence":"hourly"}`, not: []string{"zpool"}, said: "refused"}, + {name: "a failed start fails the run", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub: no such pool"} + }, ran: []string{"zpool scrub -- tank"}, wantErr: true}, + } + for _, c := range cases { + f, m, st, served := runFake(t, c.conf) + if c.setup != nil { + c.setup(f, st) + } + said, err := m.RunScheduled(context.Background(), served) + if (err != nil) != c.wantErr { + t.Errorf("%s: error %v", c.name, err) + } + for _, r := range c.ran { + if len(ran(f, r)) != 1 { + t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls) + } + } + for _, n := range c.not { + if len(ran(f, n)) != 0 { + t.Errorf("%s: ran %v", c.name, ran(f, n)) + } + } + if c.event != "" { + kind, reason, _ := strings.Cut(c.event, "/") + if len(f.emitted) != 1 || f.emitted[0]["_type"] != kind || f.emitted[0]["reason"] != reason { + t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event) + } + } + if _, err := os.Stat(served); (err == nil) != c.served { + t.Errorf("%s: served file present %v, want %v", c.name, err == nil, c.served) + } + if !strings.Contains(said, c.said) { + t.Errorf("%s: said %q, want %q", c.name, said, c.said) + } + if _, ok := st["schedule/apply"]; ok { + t.Errorf("%s: the hand act's key stayed", c.name) + } + } + // The lock held by another: nothing run, said. + f, m, _, served := runFake(t, scrubConf("tank", "monthly", "Mon", "00:00")) + m.Lock = func(string, bool) (func(), bool, error) { return nil, false, nil } + if said, err := m.RunScheduled(context.Background(), served); err != nil || !strings.Contains(said, SkipLockHeld) || + len(f.emitted) != 1 || f.emitted[0]["reason"] != SkipLockHeld || len(ran(f, "zpool")) != 0 { + t.Errorf("lock held: %q %v %v %v", said, err, f.emitted, f.calls) + } +} + +// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its +// last run a success. +func scheduled(t *testing.T) *fake { + f := withZFS(t) + f.files[ConfigFile] = scrubConf("tank", "weekly", "Mon", "00:00") + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + f.on(line("systemctl", TimerShowArgv(RunTimer)), + "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") + f.on(line("systemctl", ServiceShowArgv(RunService)), + "ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n") + return f +} + +func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) { + f := scheduled(t) + got, err := f.machine().Scrubs(context.Background(), false) + if err != nil { + t.Fatal(err) + } + a := got.(ScrubAnswer) + s := a.Schedule + if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-run.timer" || s.TimerState != "active" || + s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" { + t.Fatalf("schedule: %+v", s) + } + f = withZFS(t) + got, _ = f.machine().Scrubs(context.Background(), false) + if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") { + t.Errorf("no config: %+v", a) + } +} + +func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) { + f := scheduled(t) + h := f.machine().Health(context.Background()) + if !h.Healthy { + t.Fatalf("%+v", h) + } + w := strings.Join(h.Warnings, " ") + // tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin. + if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") || + !strings.Contains(w, "The scheduled scrub of tank due 2026-10-05 00:00 has not started") { + t.Errorf("warnings: %v", h.Warnings) + } + f.on(line("systemctl", ServiceShowArgv(RunService)), + "ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n") + f.on(line("systemctl", TimerShowArgv(RunTimer)), + "ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n") + w = strings.Join(f.machine().Health(context.Background()).Warnings, " ") + for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-run.timer is inactive and disabled"} { + if !strings.Contains(w, want) { + t.Errorf("lacks %q: %s", want, w) + } + } +} + +func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) { + for act, cmd := range map[string]string{ + "start": "sudo -n zpool scrub -- tank", + "pause": "sudo -n zpool scrub -p -- tank", + "stop": "sudo -n zpool scrub -s -- tank", + } { + f := withZFS(t) + f.on(cmd, "") + a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked") + if err != nil { + t.Fatalf("%s: %v", act, err) + } + if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done { + t.Errorf("%s: %+v", act, a) + } + if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" { + t.Errorf("%s recorded: %+v", act, f.emitted) + } + } + f := withZFS(t) + for _, c := range []struct{ act, pool, why, says string }{ + {"start", "nope", "why", "no pool"}, + {"start", "tank", "", "say why"}, + {"start", "-s", "why", "no pool"}, + {"destroy", "tank", "why", "not an act"}, + } { + if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%+v: %v", c, err) + } + } + if n := len(ran(f, "sudo")); n != 0 { + t.Errorf("a refusal ran %v", f.calls) + } + f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true} + if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") { + t.Errorf("timed out: %v", err) + } +} + +// Third review: the scheduled run's state is root's alone — a folder of its own, 0700 — so the operator account, or anything it runs, cannot make the run write elsewhere. +func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) { + dir := filepath.Join(t.TempDir(), "zfs-scrub") + path := filepath.Join(dir, "scrub-served.json") + if err := writeServed(path, map[string]int64{"tank": 1}); err != nil { + t.Fatal(err) + } + if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() { + t.Errorf("the folder: %v %v", fi.Mode(), err) + } + // A folder open to others: refused. (The refusal of a link is not run here: no test of this module makes a + // symbolic link, the mesh's hard rule.) + open := filepath.Join(t.TempDir(), "open") + os.Mkdir(open, 0o755) + if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { + t.Errorf("written into a folder others can read") + } + if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) { + t.Errorf("the run's state shares the account's folder") + } +} diff --git a/modules/zfs/cmd/zfs-tools/schedule.go b/modules/zfs/cmd/zfs-tools/schedule.go index c9e6e4eb8..50c5c217d 100644 --- a/modules/zfs/cmd/zfs-tools/schedule.go +++ b/modules/zfs/cmd/zfs-tools/schedule.go @@ -15,21 +15,19 @@ import ( // The scrub schedule the module owns. // -// The mesh writes ScrubConfig from the assignment's settings. The node-engine runs the module's -// scheduled process, `zfs-tools scrub-run`, from its own timer (ScrubTimer) every quarter of an hour, and -// the run decides from the configuration whether a scrub is due: the last scheduled time not after now -// (the slot) not yet served. Due, it starts the pool's scrub, or waits for one already running, as -// upstream's zfs-scrub@.service does; it never pauses or stops one, and nothing stops the unit to pause -// it (the node-engine writes no ExecStop). So no setting value ever reaches a unit file: the day and the -// time are read and checked here, and an invalid one is refused before anything runs. +// The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled +// process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the +// config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts +// the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the +// time are read and checked by the parse (config.go), so no setting value ever reaches a unit file. // // The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule // scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269). -// ScrubTimer and ScrubService are the node-engine's units for the scheduled run (the process's name). +// RunTimer and RunService are the node-engine's units for the scheduled run (the process's name). const ( - ScrubTimer = "zfs-mesh-scrub.timer" - ScrubService = "zfs-mesh-scrub.service" + RunTimer = "zfs-mesh-run.timer" + RunService = "zfs-mesh-run.service" ) // ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's @@ -42,9 +40,6 @@ const ServedFile = "/var/lib/zfs-scrub/scrub-served.json" // fires every quarter of an hour. const MissedAfter = time.Hour -// ScrubConfig is the schedule as the mesh wrote it from the settings. -const ScrubConfig = "/etc/zfs-tools/scrub.conf" - // Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy // pool, a monthly run on the first given weekday of a month (28 or 35 days apart). var cadenceAllows = map[string]time.Duration{ @@ -81,33 +76,22 @@ var ( clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`) ) -// ParseScrubConfig reads the schedule: pool, cadence (weekly or monthly), day (Mon … Sun) and time (HH:MM). -func ParseScrubConfig(text string) (*Schedule, error) { - v := map[string]string{} - for _, l := range strings.Split(text, "\n") { - l = strings.TrimSpace(l) - if l == "" || strings.HasPrefix(l, "#") { - continue - } - k, val, ok := strings.Cut(l, "=") - if !ok { - return nil, fmt.Errorf("%s: %q is not key=value", ScrubConfig, l) - } - v[strings.TrimSpace(k)] = strings.TrimSpace(val) +// ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the +// values are not all set (the module then has no scrub schedule, §11: not managed until set). +func ScheduleOf(c *Config) *Schedule { + if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" { + return nil } - s := &Schedule{Pool: v["pool"], Cadence: v["cadence"], Day: v["day"], Time: v["time"]} - switch { - case !poolName.MatchString(s.Pool): - return nil, fmt.Errorf("the scrub-pool setting %q is not a pool's name", s.Pool) - case cadenceAllows[s.Cadence] == 0: - return nil, fmt.Errorf("the scrub-cadence setting is %q; it is weekly or monthly", s.Cadence) - case !weekday.MatchString(s.Day): - return nil, fmt.Errorf("the scrub-day setting is %q; it is Mon, Tue, Wed, Thu, Fri, Sat or Sun", s.Day) - case !clock.MatchString(s.Time): - return nil, fmt.Errorf("the scrub-time setting is %q; it is HH:MM, 00:00 to 23:59", s.Time) + return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService} +} + +// TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is +// not set. +func TrimScheduleOf(c *Config) *Schedule { + if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" { + return nil } - s.Timer, s.Service = ScrubTimer, ScrubService - return s, nil + return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService} } var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday, @@ -150,13 +134,6 @@ func (s *Schedule) NextSlot(now time.Time) time.Time { return time.Time{} } -// WaitArgv waits for a pool's running scrub to end. -func WaitArgv(pool string) []string { return []string{"wait", "-t", "scrub", "--", pool} } - -// WaitBound is how long the scheduled run waits for a scrub: past it the run ends and leaves the scrub -// running; a scrub never pauses for it. -const WaitBound = 7 * 24 * time.Hour - // readServed is the last slot served per pool; none when the file is not there or not readable. func (m *Machine) readServed(path string) map[string]int64 { out := map[string]int64{} @@ -220,72 +197,6 @@ func startedSince(p *PoolStatus, slot time.Time) bool { return sc != nil && sc.Started != nil && !sc.Started.Before(slot) } -// ScrubRun is the scheduled run: a due scrub started, or one running waited for. It answers what it did in -// a line, and fails on an invalid schedule, which it never acts on. -func (m *Machine) ScrubRun(ctx context.Context, servedPath string) (string, error) { - raw, err := m.ReadFile(ScrubConfig) - if err != nil { - return "", fmt.Errorf("%s could not be read: %v", ScrubConfig, err) - } - s, err := ParseScrubConfig(string(raw)) - if err != nil { - return "", fmt.Errorf("refused, nothing run: %v", err) - } - now := m.Now() - slot := s.Slot(now) - served := m.readServed(servedPath) - if served[s.Pool] >= slot.Unix() { - return fmt.Sprintf("the scrub of %s scheduled %s was served", s.Pool, slot.Format("2006-01-02 15:04")), nil - } - st, err := m.Status(ctx) - if err != nil { - return "", err - } - p := statusOf(st, s.Pool) - if p == nil { - return "", fmt.Errorf("the scheduled pool %s is not imported here", s.Pool) - } - mark := func() error { - served[s.Pool] = slot.Unix() - return writeServed(servedPath, served) - } - if startedSince(p, slot) && p.Scrub().State != "running" { - return fmt.Sprintf("the scrub of %s scheduled %s ran already", s.Pool, slot.Format("2006-01-02 15:04")), mark() - } - for _, r := range p.Resilvers() { - if r.State == "running" { - return fmt.Sprintf("a resilver of %s runs; the scrub scheduled %s waits for the next run after it", s.Pool, - slot.Format("2006-01-02 15:04")), nil - } - } - started := false - if sc := p.Scrub(); sc == nil || sc.State != "running" { - _, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", s.Pool)...) - switch { - case err == nil: - started = true - case strings.Contains(err.Error(), "currently scrubbing"): - // One began since status was read: it serves the slot, and is waited for like any other. - default: - return "", fmt.Errorf("the scheduled scrub of %s could not start: %v", s.Pool, err) - } - } - if err := mark(); err != nil { - return "", fmt.Errorf("the scrub of %s runs, and the served slot could not be kept: %v", s.Pool, err) - } - wctx, cancel := context.WithTimeout(ctx, WaitBound) - defer cancel() - _, werr := m.AsRoot(wctx, "zpool", WaitArgv(s.Pool)...) - what := "waited for the scrub of " + s.Pool + " already running" - if started { - what = "started the scrub of " + s.Pool + " scheduled " + slot.Format("2006-01-02 15:04") + ", and waited for it" - } - if werr != nil { - return what + "; the wait ended without its end (" + werr.Error() + "), and the scrub goes on", nil - } - return what + " to finish", nil -} - // TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it // fires next and when it last did, as Unix times. func TimerShowArgv(unit string) []string { @@ -320,15 +231,19 @@ func unixStamp(s string) *time.Time { } // ScheduleNow is the schedule with what the service manager says of it; nil when the module has no -// schedule on this machine (no configuration written). +// schedule on this machine (no config file, or no cadence, day and time set). func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) { - raw, err := m.ReadFile(ScrubConfig) + raw, err := m.ReadFile(ConfigFile) if err != nil { return nil, nil } - s, err := ParseScrubConfig(string(raw)) - if err != nil { - return nil, err + c, refused := ParseConfig(raw, m.memMiB()) + if len(refused) > 0 { + return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason) + } + s := ScheduleOf(c) + if s == nil { + return nil, nil } if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil { s.Note = "the service manager did not say what the timer is: " + err.Error() diff --git a/modules/zfs/cmd/zfs-tools/scrub.go b/modules/zfs/cmd/zfs-tools/scrub.go index 85e9a3b4e..aa79acff7 100644 --- a/modules/zfs/cmd/zfs-tools/scrub.go +++ b/modules/zfs/cmd/zfs-tools/scrub.go @@ -141,7 +141,7 @@ func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) { case serr != nil: a.Said = append(a.Said, "The scrub schedule's settings are not valid: "+serr.Error()+".") case sched == nil: - a.Said = append(a.Said, "The module has no scrub schedule on this machine (no "+ScrubConfig+").") + a.Said = append(a.Said, "The module has no scrub schedule on this machine (no cadence, day and time set in "+ConfigFile+").") default: a.Schedule = sched line := fmt.Sprintf("%s scrub of %s, %s at %s (monthly: the first %s of the month), decided by the run %s fires "+ diff --git a/modules/zfs/cmd/zfs-tools/scrubrun_test.go b/modules/zfs/cmd/zfs-tools/scrubrun_test.go deleted file mode 100644 index 857256e07..000000000 --- a/modules/zfs/cmd/zfs-tools/scrubrun_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package main - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -// The scheduled run (`zfs-tools scrub-run`, second review of mesh-catalog #147): the node-engine's timer -// fires it every quarter of an hour, and it decides from the configuration whether a scrub is due. It -// starts one, or waits for one already running, as upstream's zfs-scrub@.service does, and never pauses or -// stops a scrub. - -func monthlyConf(pool string) string { - return "pool=" + pool + "\ncadence=monthly\nday=Mon\ntime=00:00\n" -} - -func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) { - loc := time.Local - at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) } - cases := []struct { - conf string - now time.Time - want time.Time - }{ - {"pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n", at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, - {"pool=tank\ncadence=weekly\nday=Fri\ntime=13:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)}, - {"pool=tank\ncadence=weekly\nday=Fri\ntime=11:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)}, - {monthlyConf("tank"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, - {monthlyConf("tank"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)}, - {"pool=tank\ncadence=monthly\nday=Thu\ntime=06:00\n", at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)}, - } - for _, c := range cases { - s, err := ParseScrubConfig(c.conf) - if err != nil { - t.Fatal(err) - } - if got := s.Slot(c.now); !got.Equal(c.want) { - t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want) - } - if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) { - t.Errorf("%q at %s: next slot %s", c.conf, c.now, next) - } - } -} - -// root runs the scheduled run: no sudo. -func rootFake(t *testing.T, conf string) (*fake, string) { - f := withZFS(t) - f.uid = 0 - f.files[ScrubConfig] = conf - dir := filepath.Join(t.TempDir(), "zfs-scrub") - return f, filepath.Join(dir, "scrub-served.json") -} - -func TestADueScrubIsStartedThenWaitedForAndMarkedServed(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on("zpool scrub -- tank", "") - f.on("zpool wait -t scrub -- tank", "") - said, err := f.machine().ScrubRun(context.Background(), served) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(ran(f, "zpool scrub"), "|") + "/" + strings.Join(ran(f, "zpool wait"), "|"); got != "zpool scrub -- tank/zpool wait -t scrub -- tank" { - t.Errorf("ran %q", got) - } - if !strings.Contains(said, "started") { - t.Errorf("said %q", said) - } - // Served: the next fire within the same slot runs nothing. - f.calls = nil - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 { - t.Errorf("a served slot ran %v", f.calls) - } -} - -func TestAScrubInProgressIsWaitedForNeverPausedOrStarted(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt")) - f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tONLINE\t1.00\n") - f.on("zpool wait -t scrub -- tank", "") - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if strings.Join(f.calls, "|") == "" || len(ran(f, "zpool scrub")) != 0 || len(ran(f, "zpool wait -t scrub -- tank")) != 1 { - t.Errorf("ran %v", f.calls) - } -} - -func TestAStartRefusedBecauseAScrubJustBeganIsWaitedForNotPaused(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing; use 'zpool scrub -s' to cancel the current scrub"} - f.on("zpool wait -t scrub -- tank", "") - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - for _, c := range f.calls { - if strings.Contains(c, "scrub -p") || strings.Contains(c, "scrub -s") { - t.Errorf("the run paused or stopped a scrub: %s", c) - } - } - if len(ran(f, "zpool wait -t scrub -- tank")) != 1 { - t.Errorf("not waited for: %v", f.calls) - } -} - -func TestAResilverDefersTheScrubWithoutServingTheSlot(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) - f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tDEGRADED\t1.00\n") - said, err := f.machine().ScrubRun(context.Background(), served) - if err != nil { - t.Fatal(err) - } - if len(ran(f, "zpool scrub")) != 0 || !strings.Contains(said, "resilver") { - t.Errorf("ran %v, said %q", f.calls, said) - } - if _, err := os.Stat(served); err == nil { - t.Errorf("the slot was marked served while the scrub waits for the resilver") - } -} - -func TestAScrubAlreadyStartedInTheSlotServesItWithoutAStateFile(t *testing.T) { - // tank's scrub started 2026-09-13 00:24 (finished 09-14); a weekly Sunday-00:00 slot of 09-13 is served - // by it, so a run on 09-15 starts nothing. - f, served := rootFake(t, "pool=tank\ncadence=weekly\nday=Sun\ntime=00:00\n") - f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local) - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 { - t.Errorf("ran %v", f.calls) - } -} - -func TestAnInvalidScheduleRunsNothingAndFails(t *testing.T) { - for _, conf := range []string{ - "pool=tank\ncadence=monthly\nday=Mon\ntime=*:*\n", - "pool=tank\ncadence=monthly\nday=Mon..Fri\ntime=00:00\n", - "pool=tank\ncadence=monthly\nday=*\ntime=00:00\n", - "pool=tank\ncadence=hourly\nday=Mon\ntime=00:00\n", - "pool=tank\ncadence=monthly\nday=Mon\ntime=0:00\n", - } { - f, served := rootFake(t, conf) - if _, err := f.machine().ScrubRun(context.Background(), served); err == nil { - t.Errorf("%q: not refused", conf) - } - if len(ran(f, "zpool scrub"))+len(ran(f, "zpool wait")) != 0 { - t.Errorf("%q ran %v", conf, f.calls) - } - } -} - -func TestAScheduledScrubThatDidNotStartIsAWarning(t *testing.T) { - f := withZFS(t) - f.files[ScrubConfig] = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n" - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") - h := f.machine().Health(context.Background()) - if !strings.Contains(strings.Join(h.Warnings, " "), "The scheduled scrub of tank due 2026-10-05 00:00 has not started") { - t.Errorf("warnings: %v", h.Warnings) - } -} - -// Third review: the scheduled run's state is root's alone — a folder of its own, 0700, never through a -// link — so the operator account, or anything it runs, cannot make the run write elsewhere. -func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) { - dir := filepath.Join(t.TempDir(), "zfs-scrub") - path := filepath.Join(dir, "scrub-served.json") - if err := writeServed(path, map[string]int64{"tank": 1}); err != nil { - t.Fatal(err) - } - if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() { - t.Errorf("the folder: %v %v", fi.Mode(), err) - } - // The file replaced by a link to somewhere else: refused, and the other place untouched. - victim := filepath.Join(t.TempDir(), "victim") - os.WriteFile(victim, []byte("keep"), 0o644) - os.Remove(path) - os.Symlink(victim, path) - if err := writeServed(path, map[string]int64{"tank": 2}); err == nil { - t.Errorf("written through a link") - } - if b, _ := os.ReadFile(victim); string(b) != "keep" { - t.Errorf("the link's target was changed: %q", b) - } - // The folder itself a link, or open to others: refused. - linked := filepath.Join(t.TempDir(), "linked") - os.Symlink(t.TempDir(), linked) - if err := writeServed(filepath.Join(linked, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { - t.Errorf("written into a linked folder") - } - open := filepath.Join(t.TempDir(), "open") - os.Mkdir(open, 0o755) - if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { - t.Errorf("written into a folder others can read") - } - if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) { - t.Errorf("the run's state shares the account's folder") - } -} diff --git a/modules/zfs/cmd/zfs-tools/scrubschedule_test.go b/modules/zfs/cmd/zfs-tools/scrubschedule_test.go deleted file mode 100644 index eaa8acf6d..000000000 --- a/modules/zfs/cmd/zfs-tools/scrubschedule_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" -) - -const scrubConf = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n" - -// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its -// last run a success. -func scheduled(t *testing.T) *fake { - f := withZFS(t) - f.files[ScrubConfig] = scrubConf - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), - "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") - f.on(line("systemctl", ServiceShowArgv(ScrubService)), - "ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n") - return f -} - -func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) { - f := scheduled(t) - got, err := f.machine().Scrubs(context.Background(), false) - if err != nil { - t.Fatal(err) - } - a := got.(ScrubAnswer) - s := a.Schedule - if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-scrub.timer" || s.TimerState != "active" || - s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" { - t.Fatalf("schedule: %+v", s) - } - if want := s.NextSlot(f.now).Format("2006-01-02 15:04"); !strings.Contains(strings.Join(a.Said, "\n"), "next scheduled scrub of tank: "+want) || want != "2026-10-12 00:00" { - t.Errorf("said: %v", a.Said) - } -} - -func TestAScheduleThatIsNotSetIsSaidAndAnInvalidOneRefused(t *testing.T) { - f := withZFS(t) - got, _ := f.machine().Scrubs(context.Background(), false) - if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") { - t.Errorf("no config: %+v", a) - } - for _, conf := range []string{ - "pool=tank\ncadence=daily\nday=Mon\ntime=00:00\n", - "pool=tank\ncadence=weekly\nday=Monday\ntime=00:00\n", - "pool=tank\ncadence=weekly\nday=Mon\ntime=25:00\n", - "pool=tank x\ncadence=weekly\nday=Mon\ntime=00:00\n", - } { - if _, err := ParseScrubConfig(conf); err == nil { - t.Errorf("accepted %q", conf) - } - } -} - -func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) { - f := scheduled(t) - h := f.machine().Health(context.Background()) - if !h.Healthy { - t.Fatalf("%+v", h) - } - w := strings.Join(h.Warnings, " ") - // tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin. - if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") { - t.Errorf("warnings: %v", h.Warnings) - } - f.on(line("systemctl", ServiceShowArgv(ScrubService)), - "ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n") - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), - "ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n") - w = strings.Join(f.machine().Health(context.Background()).Warnings, " ") - for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-scrub.timer is inactive and disabled"} { - if !strings.Contains(w, want) { - t.Errorf("lacks %q: %s", want, w) - } - } -} - -func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) { - for act, cmd := range map[string]string{ - "start": "sudo -n zpool scrub -- tank", - "pause": "sudo -n zpool scrub -p -- tank", - "stop": "sudo -n zpool scrub -s -- tank", - } { - f := withZFS(t) - f.on(cmd, "") - a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked") - if err != nil { - t.Fatalf("%s: %v", act, err) - } - if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done { - t.Errorf("%s: %+v", act, a) - } - if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" { - t.Errorf("%s recorded: %+v", act, f.emitted) - } - } - f := withZFS(t) - for _, c := range []struct{ act, pool, why, says string }{ - {"start", "nope", "why", "no pool"}, - {"start", "tank", "", "say why"}, - {"start", "-s", "why", "no pool"}, - {"destroy", "tank", "why", "not an act"}, - } { - if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) { - t.Errorf("%+v: %v", c, err) - } - } - if n := len(ran(f, "sudo")); n != 0 { - t.Errorf("a refusal ran %v", f.calls) - } - f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true} - if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") { - t.Errorf("timed out: %v", err) - } -} diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 195263374..bf871502e 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -187,13 +187,13 @@ "unknown": "program" }, { - "id": "scrub-run", + "id": "run", "type": "process", - "name": "zfs-mesh-scrub", + "name": "zfs-mesh-run", "artifact": "tools", "run": [ "./zfs-tools", - "scrub-run" + "run" ], "schedule": "0,15,30,45 * * * *" }, From 316039ca95f9dc4b77c42bfe111820d0ae5a6a8a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:41:21 +0100 Subject: [PATCH 08/35] Z13 zfs-hand-acts: hand acts write their key and start the run; zfs_snapshot keeps a person's snapshots apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §23, §24, A-12, A-17, A-18 --- modules/zfs/cmd/zfs-tools/bus.go | 73 +++++++++++++ modules/zfs/cmd/zfs-tools/hand.go | 114 +++++++++++++++++++++ modules/zfs/cmd/zfs-tools/hand_test.go | 80 +++++++++++++++ modules/zfs/cmd/zfs-tools/machine.go | 10 +- modules/zfs/cmd/zfs-tools/main.go | 1 + modules/zfs/cmd/zfs-tools/manifest_test.go | 10 +- modules/zfs/cmd/zfs-tools/review_test.go | 4 +- modules/zfs/cmd/zfs-tools/snapshot.go | 18 ++-- modules/zfs/cmd/zfs-tools/snapshot_test.go | 17 +-- modules/zfs/cmd/zfs-tools/tools.go | 40 +++++++- modules/zfs/module.json | 5 + 11 files changed, 348 insertions(+), 24 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/hand.go create mode 100644 modules/zfs/cmd/zfs-tools/hand_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 8737c117e..5213f39c3 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -3,6 +3,9 @@ package main import ( "encoding/json" "os" + "strings" + "sync" + "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/process" @@ -44,5 +47,75 @@ func (m *Machine) JoinBus() func() { } m.Emit = func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) } m.States = busStates{c} + m.Call = c.Call return c.Close } + +// toolStates is the module's states as the tools bundle reaches them, through the tool runner. +type toolStates struct{} + +func (toolStates) Put(state, key string, v any) error { + _, err := stdio.State(state).Put(key, v) + return err +} +func (toolStates) Delete(state, key string) error { return stdio.State(state).Delete(key) } +func (toolStates) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +func (toolStates) Get(state, key string) (json.RawMessage, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// eventHub hands the module's events, heard once through the tool runner, to the hand acts waiting for one. +type eventHub struct { + mu sync.Mutex + started bool + waiters map[int]hubWaiter + next int +} + +type hubWaiter struct { + kinds []string + ch chan map[string]any +} + +func (h *eventHub) await(kinds []string, bound time.Duration) (map[string]any, bool) { + h.mu.Lock() + if h.waiters == nil { + h.waiters = map[int]hubWaiter{} + } + h.next++ + id, w := h.next, hubWaiter{kinds, make(chan map[string]any, 1)} + h.waiters[id] = w + first := !h.started + h.started = true + h.mu.Unlock() + defer func() { h.mu.Lock(); delete(h.waiters, id); h.mu.Unlock() }() + if first { + _ = stdio.Subscribe("#", func(e stdio.Envelope) error { + var body map[string]any + _ = json.Unmarshal(e.Body, &body) + h.mu.Lock() + defer h.mu.Unlock() + for _, w := range h.waiters { + for _, k := range w.kinds { + if e.Key == k || strings.HasSuffix(e.Key, "."+k) { + select { + case w.ch <- body: + default: + } + } + } + } + return nil + }) + } + select { + case ev := <-w.ch: + return ev, true + case <-time.After(bound): + return nil, false + } +} diff --git a/modules/zfs/cmd/zfs-tools/hand.go b/modules/zfs/cmd/zfs-tools/hand.go new file mode 100644 index 000000000..a85c28704 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/hand.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" +) + +// The hand acts (design 42 zfs-owns-zfs §23, §24; A-12, A-17, A-18). +// +// A hand act writes its act into the state `schedule` under its own key {by, why, at} and starts the unit of +// the scheduled run through `/node-service-manager.start` with {scope, unit}; the run reads the key, +// acts under the same lock and guards, and deletes it. The tool waits at most HandWait for the act's event; past +// it, it answers "started", naming the act and that `zfs_module_events` reads its outcome, and never "done" for +// what it did not see end. The `why` is kept in the key. + +// HandWait is how long a hand act's tool waits for the act's event (A-12). +const HandWait = 30 * time.Second + +// The acts, as the keys of the state `schedule`. +const ( + ActSnapshots = "snapshots" + ActPrune = "prune" + ActApply = "apply" +) + +// handEnds are the events that end each act. +var handEnds = map[string][]string{ + ActSnapshots: {EventTakenPerson, EventSkippedPerson, EventRunSkipped}, + ActPrune: {EventRemovedPerson, EventSkippedPerson, EventRunSkipped}, + ActApply: {EventApplied, EventRefused, EventWaiting}, +} + +// StartArgs are what `node-service-manager.start` is called with to run the scheduled run now. +func StartArgs() map[string]any { return map[string]any{"scope": "system", "unit": RunService} } + +// HandAnswer is what a hand act's tool answers. +type HandAnswer struct { + Act string `json:"act"` + // Outcome is "seen" (the act's event came) or "started" (it was started, its end not seen). + Outcome string `json:"outcome"` + Event map[string]any `json:"event,omitempty"` + Said string `json:"said"` +} + +func whyOf(why string) error { + switch { + case why == "": + return fmt.Errorf("say why: a hand act is a person's, and the why is kept with it") + case strings.ContainsAny(why, "\n\r") || len(why) > MaxWhy: + return fmt.Errorf("say why in one line of at most %d characters", MaxWhy) + } + return nil +} + +// HandAct starts one act of the scheduled run by hand. +func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, error) { + a := HandAnswer{Act: act} + if _, ok := handEnds[act]; !ok { + return a, fmt.Errorf("%q is not an act of the run: snapshots, prune or apply", act) + } + if err := whyOf(why); err != nil { + return a, err + } + if m.States == nil || m.Call == nil { + return a, fmt.Errorf("this process reaches neither the module's state nor the service manager, so %s was not started", act) + } + if err := m.States.Put(StateSchedule, act, map[string]any{"by": by, "why": why, "at": m.Now().UTC()}); err != nil { + return a, fmt.Errorf("the act %s could not be written to the state %s: %v", act, StateSchedule, err) + } + if _, err := m.Call("seat:node-service-manager.start@"+m.Node, StartArgs()); err != nil { + _ = m.States.Delete(StateSchedule, act) + return a, fmt.Errorf("the service manager did not start %s: %v", RunService, err) + } + if m.Await != nil { + if ev, ok := m.Await(handEnds[act], HandWait); ok { + a.Outcome, a.Event = "seen", ev + a.Said = fmt.Sprintf("The act %s ran: %v.", act, ev) + return a, nil + } + } + a.Outcome = "started" + a.Said = fmt.Sprintf("Started the act %s (%s) and did not see it end within %s: zfs_module_events reads its outcome.", act, RunService, HandWait) + return a, nil +} + +// Pause writes `paused` {by, why, at} in the state `schedule`: the run then takes and removes nothing (§24). +func (m *Machine) Pause(by, why string) error { + if err := whyOf(why); err != nil { + return err + } + if m.States == nil { + return fmt.Errorf("this process reaches no state, so the schedule was not paused") + } + return m.States.Put(StateSchedule, KeyPaused, map[string]any{"by": by, "why": why, "at": m.Now().UTC()}) +} + +// Resume deletes `paused`. +func (m *Machine) Resume() error { + if m.States == nil { + return fmt.Errorf("this process reaches no state, so the schedule was not resumed") + } + return m.States.Delete(StateSchedule, KeyPaused) +} + +// Paused says whether the schedule is paused. +func (m *Machine) Paused() bool { + if m.States == nil { + return false + } + _, ok, err := m.States.Get(StateSchedule, KeyPaused) + return err == nil && ok +} diff --git a/modules/zfs/cmd/zfs-tools/hand_test.go b/modules/zfs/cmd/zfs-tools/hand_test.go new file mode 100644 index 000000000..c4f685e86 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/hand_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +// §23, §24, A-12, A-17, A-18: a hand act writes its key and starts the run's unit with {scope, unit}; the tool +// answers from the act's event when it comes within the wait, else "started", never "done"; a failed start +// leaves no key; pause writes `paused`, resume deletes it. +func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { + cases := []struct { + name, act, why string + event map[string]any + callErr error + outcome string + err string + keyLeft bool + waited []string + }{ + {name: "event seen", act: ActSnapshots, why: "before the move", event: map[string]any{"snapshot": "tank@mesh-hourly"}, outcome: "seen", keyLeft: true, + waited: handEnds[ActSnapshots]}, + {name: "event not seen in time", act: ActPrune, why: "free space", outcome: "started", keyLeft: true, waited: handEnds[ActPrune]}, + {name: "apply", act: ActApply, why: "now", event: map[string]any{"key": "hostid"}, outcome: "seen", keyLeft: true, waited: handEnds[ActApply]}, + {name: "start refused", act: ActSnapshots, why: "x", callErr: errors.New("refused"), err: "did not start"}, + {name: "no why", act: ActSnapshots, why: "", err: "say why"}, + {name: "unknown act", act: "destroy", why: "x", err: "not an act"}, + } + for _, c := range cases { + f := newFake(t) + m := f.machine() + st := memStates{} + m.States, m.Node = st, "ace" + var key string + var body any + m.Call = func(k string, b any) (json.RawMessage, error) { key, body = k, b; return nil, c.callErr } + var bound time.Duration + var kinds []string + m.Await = func(k []string, d time.Duration) (map[string]any, bool) { + kinds, bound = k, d + return c.event, c.event != nil + } + a, err := m.HandAct(context.Background(), c.act, "caller@node", c.why) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) { + t.Errorf("%s: %v", c.name, err) + } + } else if err != nil || a.Outcome != c.outcome { + t.Errorf("%s: %+v %v", c.name, a, err) + } + if _, left := st["schedule/"+c.act]; left != c.keyLeft { + t.Errorf("%s: key left %v, want %v", c.name, left, c.keyLeft) + } + if c.err == "" || c.callErr != nil { + args, _ := body.(map[string]any) + if key != "seat:node-service-manager.start@ace" || args["unit"] != "zfs-mesh-run.service" || args["scope"] != "system" || len(args) != 2 { + t.Errorf("%s: called %s with %v", c.name, key, body) + } + } + if c.err == "" && (bound != 30*time.Second || strings.Join(kinds, ",") != strings.Join(c.waited, ",")) { + t.Errorf("%s: waited %v for %v", c.name, bound, kinds) + } + if c.outcome == "started" && strings.Contains(a.Said, "done") { + t.Errorf("%s: said done for what it did not see: %s", c.name, a.Said) + } + } + f := newFake(t) + m := f.machine() + m.States = memStates{} + if err := m.Pause("p@n", "maintenance"); err != nil || !m.Paused() { + t.Fatal(err, m.Paused()) + } + if err := m.Resume(); err != nil || m.Paused() { + t.Fatal(err, m.Paused()) + } +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 49ed8f0aa..86ef58a42 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -117,6 +117,14 @@ type Machine struct { // Lock takes an exclusive lock on a file: waiting for it, or answering at once whether it was got. The // release gives it up. Lock func(path string, wait bool) (release func(), got bool, err error) + // Call invokes a tool or a seat verb as this module (`seat:.@`, `.`); nil where + // this process reaches none. + Call func(key string, body any) (json.RawMessage, error) + // Node is this machine's name, for a seat verb of its own. + Node string + // Await waits at most the bound for an event of the module whose kind is one of kinds, and answers it; false + // when none came. + Await func(kinds []string, bound time.Duration) (map[string]any, bool) // Warnings are what the apply saw that is no failure (a change the bus did not take, a parameter awaiting // a load). Warnings []string @@ -133,7 +141,7 @@ type Machine struct { // Here is the machine this process runs on. func Here() *Machine { - return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, WriteFile: writeLive, Lock: FileLock, Errors: LoadErrorMemory(ErrorsFile)} + return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Node: os.Getenv("MESH_NODE"), WriteFile: writeLive, Lock: FileLock, Errors: LoadErrorMemory(ErrorsFile)} } // NotApplicable is the answer of a machine without ZFS: the module does not apply here. diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 1056e65e3..55898bc45 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -67,6 +67,7 @@ func main() { } m := Here() m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) } + m.States, m.Call, m.Await = toolStates{}, stdio.Ask, (&eventHub{}).await // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. if err := stdio.Serve("", Tools(m)); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 079bbc266..e024087e6 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -35,19 +35,23 @@ func readManifest(t *testing.T) (manifest, string) { // The manifest lists exactly the tools the bundle serves; every one but zfs_snapshot says it only reads, // and zfs_snapshot says it acts. Each says what it replaces (novox/hq ADR 0245), but the health check, // which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121). +// handTools are the tools that start or steer the scheduled run, which replace no command. +var handTools = map[string]bool{"zfs_snapshots_run_now": true, "zfs_snapshots_prune_now": true, "zfs_config_apply": true, + "zfs_snapshots_pause": true, "zfs_snapshots_resume": true} + func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) { m, raw := readManifest(t) var served []string for _, tl := range Tools(newFake(t).machine()) { served = append(served, tl.Name) want := "(r)" - if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") { + if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || handTools[tl.Name] { want = "(a)" } if !strings.HasSuffix(tl.Description, want) { t.Errorf("%s does not end with %s", tl.Name, want) } - if tl.Name != "zfs_health" && len(m.Replaces[tl.Name]) == 0 { + if tl.Name != "zfs_health" && !handTools[tl.Name] && len(m.Replaces[tl.Name]) == 0 { t.Errorf("%s says nothing of what it replaces", tl.Name) } for _, c := range m.Replaces[tl.Name] { @@ -68,7 +72,7 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T if len(m.Claims) != 0 { t.Errorf("the module claims no seat: %v", m.Claims) } - if strings.Join(m.Emits, ",") != SnapshotEvent+","+ScrubEvent { + if strings.Join(m.Emits, ",") != "snapshot.taken,"+ScrubEvent { t.Errorf("emits %v", m.Emits) } if hit := regexp.MustCompile(`\b(g14|novox|ace|shanks|jochen)\b|/home/`).FindString(strings.ReplaceAll(raw, "novox/hq", "")); hit != "" { diff --git a/modules/zfs/cmd/zfs-tools/review_test.go b/modules/zfs/cmd/zfs-tools/review_test.go index 8d8408b19..08fb09cfe 100644 --- a/modules/zfs/cmd/zfs-tools/review_test.go +++ b/modules/zfs/cmd/zfs-tools/review_test.go @@ -89,14 +89,14 @@ func TestTheSnapshotHasNoRecursiveAndSaysWhatItWillHold(t *testing.T) { func TestASnapshotThatTimedOutIsOutcomeUnknownUnlessTheListShowsIt(t *testing.T) { f := withZFS(t) f.answers[takeLine] = Ran{Code: -1, TimedOut: true, Err: "did not answer within its time"} - _, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + _, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_snapshots") || strings.Contains(err.Error(), "NOT DONE") { t.Errorf("timed out, not shown: %v", err) } f = withZFS(t) snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t1") f.answers[takeLine] = Ran{Code: -1, TimedOut: true} - a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err != nil || !a.Done { t.Errorf("timed out and shown by the list: done (%v, %+v)", err, a) } diff --git a/modules/zfs/cmd/zfs-tools/snapshot.go b/modules/zfs/cmd/zfs-tools/snapshot.go index 94d86e199..64f1c4275 100644 --- a/modules/zfs/cmd/zfs-tools/snapshot.go +++ b/modules/zfs/cmd/zfs-tools/snapshot.go @@ -9,9 +9,9 @@ import ( "time" ) -// SnapshotEvent is the event each attempted snapshot is published as (module.json's emits): the record -// of the act, beside the pool's own history, which ZFS writes for every snapshot. -const SnapshotEvent = "snapshot.taken" +// MeshPrefix begins the name of every ZFS snapshot the scheduled run takes (§21): a person's snapshot never +// does, so the run's snapshots and the person's stay apart (§23). +const MeshPrefix = "mesh-" // The one act of this module: a snapshot of one dataset, by a person, with why. It adds a snapshot and // changes nothing else: no destroy, rollback, set or send exists in this module (novox/hq ADR 0051, the @@ -71,7 +71,7 @@ type SnapshotAnswer struct { // TakeSnapshot takes dataset@name. It refuses an unknown dataset, a name with anything but // [A-Za-z0-9._-], an empty why, and a snapshot that already exists. It answers what `zfs list` shows // afterwards. -func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why string) (SnapshotAnswer, error) { +func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why, by string) (SnapshotAnswer, error) { a := SnapshotAnswer{Dataset: dataset, Name: name, Why: why, Made: []Made{}, Others: []string{}, Pool: poolOf(dataset)} switch { case dataset == "": @@ -80,6 +80,8 @@ func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why string) ( return a, fmt.Errorf("%q is not a dataset: name the filesystem or volume, and the snapshot's name in name", dataset) case name == "": return a, fmt.Errorf("name the snapshot") + case strings.HasPrefix(name, MeshPrefix): + return a, fmt.Errorf("the snapshot's name may not begin %q: those are the scheduled run's, and a person's snapshot stays apart from them", MeshPrefix) case !snapName.MatchString(name): return a, fmt.Errorf("the snapshot's name %q may hold only letters, digits, '.', '_' and '-'", name) case len(dataset)+1+len(name) > MaxName: @@ -176,7 +178,7 @@ func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why string) ( a.Said += " " + space + " " + a.Holds + "." } - a.Recorded, a.Record = m.record(a) + a.Recorded, a.Record = m.record(a, by) if !a.Recorded { a.Said += " NOT RECORDED on the bus: " + a.Record + "; the pool's own history (zpool history) has the command." } @@ -188,7 +190,7 @@ func (m *Machine) TakeSnapshot(ctx context.Context, dataset, name, why string) ( } // record publishes the act as this module's event: the names, the reason and the outcome, nothing else. -func (m *Machine) record(a SnapshotAnswer) (bool, string) { +func (m *Machine) record(a SnapshotAnswer, by string) (bool, string) { if m.Emit == nil { return false, "this process publishes no events" } @@ -196,13 +198,13 @@ func (m *Machine) record(a SnapshotAnswer) (bool, string) { if len(a.Made) > 0 { created = a.Made[0].Created } - err := m.Emit(SnapshotEvent, map[string]any{"dataset": a.Dataset, "snapshot": a.Name, + err := m.Emit(EventTakenPerson, map[string]any{"dataset": a.Dataset, "snapshot": a.Dataset + "@" + a.Name, "by": by, "why": a.Why, "done": a.Done, "outcome": a.Outcome, "made": madeNames(a.Made), "created": created, "at": m.Now().UTC().Format(time.RFC3339)}) if err != nil { return false, err.Error() } - return true, "published as the event " + SnapshotEvent + return true, "published as the event " + EventTakenPerson } // spaceAfter reads the dataset's free space and its pool's capacity after the act, and says them. diff --git a/modules/zfs/cmd/zfs-tools/snapshot_test.go b/modules/zfs/cmd/zfs-tools/snapshot_test.go index def2e34ff..b2f5b8f9b 100644 --- a/modules/zfs/cmd/zfs-tools/snapshot_test.go +++ b/modules/zfs/cmd/zfs-tools/snapshot_test.go @@ -39,7 +39,7 @@ func TestTheSnapshotCommandIsFixedWordsThroughSudoWithTheNameAfterDashDash(t *te func TestASnapshotIsAnsweredFromTheListAfterwardsAndRecorded(t *testing.T) { f := withZFS(t) snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t26938034880512") - a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "before moving the library") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "before moving the library", "tester") if err != nil { t.Fatal(err) } @@ -55,7 +55,7 @@ func TestASnapshotIsAnsweredFromTheListAfterwardsAndRecorded(t *testing.T) { if strings.Join(a.Ran, " ") != takeLine || len(ran(f, "sudo")) != 1 { t.Errorf("ran: %v; every call: %v", a.Ran, f.calls) } - if len(f.emitted) != 1 || f.emitted[0]["_type"] != SnapshotEvent || f.emitted[0]["why"] != "before moving the library" || + if len(f.emitted) != 1 || f.emitted[0]["_type"] != EventTakenPerson || f.emitted[0]["why"] != "before moving the library" || f.emitted[0]["done"] != true || f.emitted[0]["made"] != "tank/media@before-move" || !a.Recorded { t.Errorf("recorded: %+v", f.emitted) } @@ -64,7 +64,7 @@ func TestASnapshotIsAnsweredFromTheListAfterwardsAndRecorded(t *testing.T) { func TestASnapshotTheListDoesNotShowIsNotAnsweredAsMade(t *testing.T) { f := withZFS(t) f.on(takeLine, "") // zfs exits 0, and the list does not change - a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err == nil || a.Done || !strings.Contains(err.Error(), "the list shows 0 of the 1") { t.Fatalf("err %v, answer %+v", err, a) } @@ -76,7 +76,7 @@ func TestASnapshotTheListDoesNotShowIsNotAnsweredAsMade(t *testing.T) { func TestARefusedSudoIsSaidAndRecorded(t *testing.T) { f := withZFS(t) f.answers[takeLine] = Ran{Code: 1, Err: "sudo: a password is required"} - _, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + _, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err == nil || !strings.Contains(err.Error(), "NOT DONE") || !strings.Contains(err.Error(), "sudo refused the operator account") { t.Fatalf("err: %v", err) } @@ -89,7 +89,7 @@ func TestAnUnrecordedActSaysSoLoudly(t *testing.T) { f := withZFS(t) snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t1") f.emitErr = errors.New("the bus did not accept it") - a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestAnUnrecordedActSaysSoLoudly(t *testing.T) { func TestAnotherSnapshotMeanwhileIsSaidNotHidden(t *testing.T) { f := withZFS(t) snapshotting(f, takeLine, "tank/media@before-move\t1791400000\t0\t1", "tank/downloads@auto-1200\t1791547201\t0\t1") - a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why") + a, err := f.machine().TakeSnapshot(context.Background(), "tank/media", "before-move", "why", "tester") if err != nil { t.Fatal(err) } @@ -116,6 +116,7 @@ func TestEveryRefusalRunsNoSnapshotCommand(t *testing.T) { {"", "x", "why", "name the dataset"}, {"tank/media@x", "y", "why", "is not a dataset"}, {"tank/media", "", "why", "name the snapshot"}, + {"tank/media", "mesh-hourly-20261109T1200Z", "why", "may not begin"}, {"tank/media", "a b", "why", "may hold only"}, {"tank/media", "x;rm -rf /", "why", "may hold only"}, {"tank/media", "$(reboot)", "why", "may hold only"}, @@ -129,7 +130,7 @@ func TestEveryRefusalRunsNoSnapshotCommand(t *testing.T) { } for _, c := range cases { f := withZFS(t) - _, err := f.machine().TakeSnapshot(context.Background(), c.dataset, c.name, c.why) + _, err := f.machine().TakeSnapshot(context.Background(), c.dataset, c.name, c.why, "tester") if c.says == "" { if len(ran(f, "sudo -n zfs snapshot -- tank/media@-r")) != 1 { t.Errorf("%q: the name follows -- and runs as a name: %v", c.name, f.calls) @@ -149,7 +150,7 @@ func TestAMachineWithoutZFSSaysTheModuleDoesNotApply(t *testing.T) { f := newFake(t) // every command: executable file not found m := f.machine() for _, tool := range Tools(m) { - if tool.Name == "zfs_health" { + if tool.Name == "zfs_health" || handTools[tool.Name] { continue } args := map[string]any{"dataset": "tank", "name": "x", "why": "y"} diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go index 483b78a54..d77989000 100644 --- a/modules/zfs/cmd/zfs-tools/tools.go +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -41,7 +41,7 @@ func Tools(m *Machine) []stdio.Tool { "answers what zfs list shows: the snapshot made, its creation time, any other snapshot that came or went " + "meanwhile, the dataset's free space and the pool's capacity. From then on the snapshot holds every block " + "of the dataset deleted or overwritten, until it is destroyed by hand. A command that ran out of time is " + - "answered from the list, or as outcome unknown. Recorded as the event snapshot.taken; ZFS keeps it in the " + + "answered from the list, or as outcome unknown. A name beginning mesh- is refused (the scheduled run's). Recorded as the event snapshot.taken.person; ZFS keeps it in the " + "pool's history too. It adds a snapshot and changes nothing else. Replaces zfs snapshot. (a)", Input: map[string]any{ "dataset": dataset("the dataset, as zfs_datasets names it"), @@ -52,7 +52,7 @@ func Tools(m *Machine) []stdio.Tool { if _, given := a["recursive"]; given { return nil, fmt.Errorf("recursive does not exist in this version: zfs_snapshot snapshots one dataset, never its children") } - ans, err := m.TakeSnapshot(ctx, text(a, "dataset"), text(a, "name"), text(a, "why")) + ans, err := m.TakeSnapshot(ctx, text(a, "dataset"), text(a, "name"), text(a, "why"), stdio.CallerOf(a)) if err != nil { if _, absentHere := err.(errAbsent); absentHere { return NotApplicable{Applicable: false, Said: err.Error()}, nil @@ -62,6 +62,30 @@ func Tools(m *Machine) []stdio.Tool { return ans, nil }, }, + handTool(m, "zfs_snapshots_run_now", ActSnapshots, "Take the scheduled ZFS snapshots now, as the scheduled run does (a person's act with why): writes the act into the state `schedule` and starts zfs-mesh-run.service through the service manager; the run takes it under the same lock and guards. Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), + handTool(m, "zfs_snapshots_prune_now", ActPrune, "Remove the scheduled ZFS snapshots past their keep count now, as the scheduled run does, only those its own ledger holds (a person's act with why). Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), + handTool(m, "zfs_config_apply", ActApply, "Converge the node to its config file now, as the apply does (a person's act with why): the same lock and guards as the apply. Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), + { + Name: "zfs_snapshots_pause", + Description: "Pause the scheduled ZFS snapshots: writes `paused` {by, why, at} in the state `schedule`; the run then takes and removes nothing until zfs_snapshots_resume. (a)", + Input: map[string]any{"why": map[string]any{"type": "string", "description": "why, in one line; kept with the act"}}, + Run: func(a map[string]any) (any, error) { + if err := m.Pause(stdio.CallerOf(a), text(a, "why")); err != nil { + return nil, err + } + return map[string]any{"paused": true}, nil + }, + }, + { + Name: "zfs_snapshots_resume", + Description: "Resume the scheduled ZFS snapshots: deletes `paused` from the state `schedule`. (a)", + Run: func(map[string]any) (any, error) { + if err := m.Resume(); err != nil { + return nil, err + } + return map[string]any{"paused": false}, nil + }, + }, { Name: "zfs_pools", Description: "Every imported pool's health in a line: its state (ONLINE, DEGRADED, FAULTED, …) with status's own " + @@ -255,3 +279,15 @@ func whole(a map[string]any, key string, fallback, least, most int) (int, error) } return n, nil } + +// handTool is one hand act as a tool. +func handTool(m *Machine, name, act, description string) stdio.Tool { + return stdio.Tool{ + Name: name, + Description: description, + Input: map[string]any{"why": map[string]any{"type": "string", "description": "why, in one line; kept with the act"}}, + Run: func(a map[string]any) (any, error) { + return m.HandAct(context.Background(), act, stdio.CallerOf(a), text(a, "why")) + }, + } +} diff --git a/modules/zfs/module.json b/modules/zfs/module.json index bf871502e..082c8dcbe 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -59,6 +59,11 @@ "zfs_scrub_start", "zfs_scrub_pause", "zfs_scrub_stop", + "zfs_snapshots_run_now", + "zfs_snapshots_prune_now", + "zfs_config_apply", + "zfs_snapshots_pause", + "zfs_snapshots_resume", "zfs_io", "zfs_arc", "zfs_events", From a357a85faa6292192cce5ab7a4a70ab57c39fa55 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:43:09 +0100 Subject: [PATCH 09/35] Z11 zfs-snapshots-take: the run takes its due ZFS snapshots per filesystem and records each in the ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §21, §26, A-19, A-20, A-30, A-31 --- modules/zfs/cmd/zfs-tools/bus.go | 42 +++ modules/zfs/cmd/zfs-tools/machine.go | 2 + modules/zfs/cmd/zfs-tools/main.go | 2 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 6 +- modules/zfs/cmd/zfs-tools/run.go | 26 ++ modules/zfs/cmd/zfs-tools/snapshots_take.go | 275 ++++++++++++++++++++ modules/zfs/cmd/zfs-tools/snapshots_test.go | 162 ++++++++++++ modules/zfs/cmd/zfs-tools/tools.go | 20 ++ modules/zfs/module.json | 1 + 9 files changed, 532 insertions(+), 4 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/snapshots_take.go create mode 100644 modules/zfs/cmd/zfs-tools/snapshots_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 5213f39c3..41ec813b3 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -8,6 +8,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -119,3 +120,44 @@ func (h *eventHub) await(kinds []string, bound time.Duration) (map[string]any, b return nil, false } } + +// busLedger is the ledger `snapshots` through the tool runner. +type busLedger struct{} + +func (busLedger) Apply(instance, operation string, body any, call string) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + _, err = stdio.BusState(buscalls.LedgerApply, buscalls.ApplyRequest{Ledger: LedgerName, Instance: instance, + Operation: operation, Body: raw, Call: call}) + return err +} + +func (busLedger) Entries() ([]LedgerEntry, error) { + raw, err := stdio.BusState(buscalls.LedgerQuery, buscalls.QueryRequest{Ledger: LedgerName, Index: "by-dataset-kind"}) + if err != nil { + return nil, err + } + var q buscalls.QueryAnswer + if err := json.Unmarshal(raw, &q); err != nil { + return nil, err + } + var out []LedgerEntry + for _, ie := range q.Entries { + raw, err := stdio.BusState(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: LedgerName, Instance: ie.Instance}) + if err != nil { + return nil, err + } + var st buscalls.InstanceState + if err := json.Unmarshal(raw, &st); err != nil { + return nil, err + } + var e LedgerEntry + if err := json.Unmarshal(st.State, &e); err != nil { + return nil, err + } + out = append(out, e) + } + return out, nil +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 86ef58a42..669aa5039 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -117,6 +117,8 @@ type Machine struct { // Lock takes an exclusive lock on a file: waiting for it, or answering at once whether it was got. The // release gives it up. Lock func(path string, wait bool) (release func(), got bool, err error) + // Ledger is the ledger `snapshots` as the tools bundle reaches it; nil in a process, which cannot (A-20). + Ledger LedgerStore // Call invokes a tool or a seat verb as this module (`seat:.@`, `.`); nil where // this process reaches none. Call func(key string, body any) (json.RawMessage, error) diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 55898bc45..0fb7282db 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -67,7 +67,7 @@ func main() { } m := Here() m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) } - m.States, m.Call, m.Await = toolStates{}, stdio.Ask, (&eventHub{}).await + m.States, m.Call, m.Ledger, m.Await = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. if err := stdio.Serve("", Tools(m)); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index e024087e6..45c15b9fb 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -37,7 +37,7 @@ func readManifest(t *testing.T) (manifest, string) { // which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121). // handTools are the tools that start or steer the scheduled run, which replace no command. var handTools = map[string]bool{"zfs_snapshots_run_now": true, "zfs_snapshots_prune_now": true, "zfs_config_apply": true, - "zfs_snapshots_pause": true, "zfs_snapshots_resume": true} + "zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true} func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) { m, raw := readManifest(t) @@ -228,7 +228,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, "DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"}, "AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"}, - "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, + "SnapshotSetArgv": {"zfs", "set"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), @@ -240,7 +240,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DeviceStatusArgv": DeviceStatusArgv("tank", true), "AllRealStatusArgv": AllRealStatusArgv(), "RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"), "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x"), - "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")} + "SnapshotSetArgv": SnapshotSetArgv("tank@mesh-hourly-x", "hourly"), "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")} for fn, argv := range verbs { for _, a := range argv { if fn != "FollowArgv" && fn != "ZgenhostidArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") { diff --git a/modules/zfs/cmd/zfs-tools/run.go b/modules/zfs/cmd/zfs-tools/run.go index 922cefdf9..a99894446 100644 --- a/modules/zfs/cmd/zfs-tools/run.go +++ b/modules/zfs/cmd/zfs-tools/run.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "sort" "strings" @@ -97,7 +98,14 @@ func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, } } for _, act := range acts { + by := m.actBy(act) switch { + case act == ActSnapshots && guard == "": + ts, err := m.TakeScheduled(ctx, c, true, by) + said = append(said, ts...) + if err != nil { + failed = append(failed, err) + } case act == "apply": s, err := m.applyLocked(ctx) said = append(said, "apply: "+s) @@ -150,6 +158,11 @@ func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, failed = append(failed, fmt.Errorf("the %s of %s was started, and the served slot could not be kept: %v", work.name, c.Pool, err)) } } + ts, err := m.TakeScheduled(ctx, c, false, "") + said = append(said, ts...) + if err != nil { + failed = append(failed, err) + } if len(said) == 0 { said = append(said, "nothing due") } @@ -166,3 +179,16 @@ func joinErrors(errs []error) error { } return fmt.Errorf("%s", strings.Join(parts, "; ")) } + +// actBy is who a hand act was by, from its key. +func (m *Machine) actBy(act string) string { + raw, ok, err := m.States.Get(StateSchedule, act) + if err != nil || !ok { + return "" + } + var v struct { + By string `json:"by"` + } + _ = json.Unmarshal(raw, &v) + return v.By +} diff --git a/modules/zfs/cmd/zfs-tools/snapshots_take.go b/modules/zfs/cmd/zfs-tools/snapshots_take.go new file mode 100644 index 000000000..977618db5 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshots_take.go @@ -0,0 +1,275 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + "time" +) + +// The run's ZFS snapshots: taking them (design 42 zfs-owns-zfs §21, A-20, A-30, A-31). +// +// Each kind whose keep count is above 0 is due at the first run of the hour (hourly), the first run after +// midnight (daily), on Monday (weekly) and on the first of the month (monthly), in the node's time zone, +// against the ledger's last entry of that kind. For every filesystem of the pool the run takes `zfs snapshot -- +// @mesh--`, never recursive, sets `mesh:schedule=` on it, and records `taken` +// through the module's own tool `zfs_snapshots_record` (a process cannot write a ledger, A-20). It reads the +// ledger through `zfs_snapshots_schedule` (A-31). It takes none it cannot record in the same run: with the bus +// down or the ledger full it takes none and says `snapshot.skipped.schedule`, and none while the pool is more +// than 80 % full. + +// FullPercent is the capacity above which no scheduled ZFS snapshot is taken. +const FullPercent = 80 + +// The tools the run calls on its own module (A-20, A-31). +const ( + ToolRecord = "zfs.zfs_snapshots_record" + ToolSchedule = "zfs.zfs_snapshots_schedule" +) + +// StampForm is the UTC stamp in a scheduled snapshot's name. +const StampForm = "20060102T1504Z" + +var meshSnapshot = regexp.MustCompile(`^mesh-(hourly|daily|weekly|monthly)-([0-9]{8}T[0-9]{4}Z)$`) + +// SnapshotSetArgv sets a user property on a snapshot; only mesh:schedule is ever set by the module. +func SnapshotSetArgv(snapshot, kind string) []string { + return []string{"set", "mesh:schedule=" + kind, "--", snapshot} +} + +// LedgerEntry is one instance of the ledger `snapshots` (A-30): the ZFS snapshot the run took. +type LedgerEntry struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + Kind string `json:"kind"` + At string `json:"at"` +} + +// LedgerInstance is the instance path of a scheduled snapshot: `snapshot/--`, the +// dataset's `/` written `~`. +func LedgerInstance(kind, stamp, dataset string) string { + return "snapshot/" + kind + "-" + stamp + "-" + strings.ReplaceAll(dataset, "/", "~") +} + +// InstanceOf is the instance of a record's snapshot, refusing a name that is not a scheduled snapshot's. +func InstanceOf(dataset, name string) (string, error) { + m := meshSnapshot.FindStringSubmatch(name) + if m == nil { + return "", fmt.Errorf("%q is not a scheduled snapshot's name (mesh--): the ledger holds only the run's", name) + } + if dataset == "" || strings.ContainsAny(dataset, "@~ ") { + return "", fmt.Errorf("%q is not a dataset's name", dataset) + } + return LedgerInstance(m[1], m[2], dataset), nil +} + +// ledgerEntries reads the ledger through the module's tool zfs_snapshots_schedule. +func (m *Machine) ledgerEntries() ([]LedgerEntry, error) { + if m.Call == nil { + return nil, fmt.Errorf("this process reaches no tool, so the ledger cannot be read") + } + raw, err := m.Call(ToolSchedule, map[string]any{}) + if err != nil { + return nil, err + } + var ans struct { + Ledger []LedgerEntry `json:"ledger"` + } + if err := json.Unmarshal(raw, &ans); err != nil { + return nil, fmt.Errorf("zfs_snapshots_schedule answered what is no ledger: %v", err) + } + return ans.Ledger, nil +} + +// record tells the ledger of a taken or removed snapshot through zfs_snapshots_record. +func (m *Machine) recordLedger(op string, e LedgerEntry) error { + if m.Call == nil { + return fmt.Errorf("this process reaches no tool, so %s was not recorded", e.Name) + } + _, err := m.Call(ToolRecord, map[string]any{"operation": op, "dataset": e.Dataset, "name": e.Name, "kind": e.Kind, "at": e.At}) + return err +} + +// keepsOf are the kinds with a keep count above 0, in the order they are said. +func keepsOf(k *Keeps) []string { + var out []string + for _, kind := range keepKinds { + if k.Of(kind) > 0 { + out = append(out, kind) + } + } + return out +} + +// dueKinds are the kinds due now: the calendar's moment of the kind, and no entry of the kind since its period began. +func dueKinds(k *Keeps, now time.Time, entries []LedgerEntry) []string { + day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + var due []string + for _, kind := range keepsOf(k) { + start := day + switch kind { + case "hourly": + start = now.Truncate(time.Hour) + case "weekly": + if now.Weekday() != time.Monday { + continue + } + case "monthly": + if now.Day() != 1 { + continue + } + } + var last time.Time + for _, e := range entries { + if at, err := time.Parse(time.RFC3339, e.At); err == nil && e.Kind == kind && at.After(last) { + last = at + } + } + if last.Before(start) { + due = append(due, kind) + } + } + return due +} + +// ledgerCap is the ledger's bound: the keep counts' sum times two instances per dataset (§26). +func ledgerCap(k *Keeps, datasets int) int { + return (k.Hourly + k.Daily + k.Weekly + k.Monthly) * 2 * datasets +} + +func (m *Machine) skippedSnapshots(person bool, pool string, capacity int, reason string) { + kind := EventSkippedSched + if person { + kind = EventSkippedPerson + } + var failed []error + if err := m.say(kind, map[string]any{"pool": pool, "capacity": capacity, "reason": reason}); err != nil { + m.failure(&failed, kind, err) + } +} + +// filesystems are the pool's filesystems, never a volume. +func (m *Machine) filesystems(ctx context.Context, pool string) ([]string, int, error) { + ds, err := m.Datasets(ctx) + if err != nil { + return nil, 0, err + } + var out []string + for _, d := range ds { + if d.Type == "filesystem" && (d.Name == pool || strings.HasPrefix(d.Name, pool+"/")) { + out = append(out, d.Name) + } + } + capacity := 0 + ps, err := m.Pools(ctx) + if err != nil { + return nil, 0, err + } + for _, p := range ps { + if p.Name == pool { + capacity = p.Capacity + } + } + return out, capacity, nil +} + +// TakeScheduled takes the due ZFS snapshots (every kind with a keep count when force: a person's act, by +// them). It answers what it did, and a failed command as the error. +func (m *Machine) TakeScheduled(ctx context.Context, c *Config, force bool, by string) ([]string, error) { + if c.Snapshots == nil || c.Snapshots.None || c.Pool == "" { + return nil, nil + } + person := force + taken := EventTakenSchedule + if person { + taken = EventTakenPerson + } else { + by = "schedule" + } + fs, capacity, err := m.filesystems(ctx, c.Pool) + if err != nil { + return nil, err + } + if m.Paused() { + m.skippedSnapshots(person, c.Pool, capacity, "paused") + return []string{"snapshots paused"}, nil + } + entries, err := m.ledgerEntries() + if err != nil { + m.skippedSnapshots(person, c.Pool, capacity, "ledger-unavailable: "+err.Error()) + return []string{"no snapshot taken: the ledger cannot be read"}, nil + } + due := keepsOf(c.Snapshots) + if !force { + due = dueKinds(c.Snapshots, m.Now(), entries) + } + if len(due) == 0 || len(fs) == 0 { + return nil, nil + } + if capacity > FullPercent { + m.skippedSnapshots(person, c.Pool, capacity, "full") + return []string{fmt.Sprintf("no snapshot taken: the pool is %d %% full", capacity)}, nil + } + if len(entries)+len(due)*len(fs) > ledgerCap(c.Snapshots, len(fs)) { + m.skippedSnapshots(person, c.Pool, capacity, "ledger-full") + return []string{"no snapshot taken: the ledger is full"}, nil + } + now := m.Now().UTC() + var said []string + var failed []error + for _, kind := range due { + for _, ds := range fs { + name := fmt.Sprintf("%s%s-%s", MeshPrefix, kind, now.Format(StampForm)) + full := ds + "@" + name + if _, err := m.AsRoot(ctx, "zfs", SnapshotArgv(ds, name)...); err != nil { + failed = append(failed, fmt.Errorf("%s: %v", full, err)) + continue + } + if _, err := m.AsRoot(ctx, "zfs", SnapshotSetArgv(full, kind)...); err != nil { + failed = append(failed, fmt.Errorf("%s was taken, and mesh:schedule could not be set: %v", full, err)) + continue + } + e := LedgerEntry{Dataset: ds, Name: name, Kind: kind, At: now.Format(time.RFC3339)} + if err := m.recordLedger("taken", e); err != nil { + failed = append(failed, fmt.Errorf("%s was taken, and the ledger did not take it: %v", full, err)) + continue + } + if err := m.say(taken, map[string]any{"snapshot": full, "by": by}); err != nil { + m.failure(&failed, taken, err) + } + said = append(said, "took "+full) + } + } + return said, joinErrors(failed) +} + +// LedgerName is the ledger's local name in the module's `requires`. +const LedgerName = "snapshots" + +// LedgerStore is the ledger `snapshots` as the tools bundle reaches it (A-20, A-30). +type LedgerStore interface { + // Apply applies an operation to an instance; call is the id of the tool call it serves. + Apply(instance, operation string, body any, call string) error + // Entries are every instance's state. + Entries() ([]LedgerEntry, error) +} + +// Record applies the run's `taken` or `removed` to the ledger, refusing a snapshot not named mesh- (A-20). +func (m *Machine) Record(op string, e LedgerEntry, call string) error { + if op != "taken" && op != "removed" { + return fmt.Errorf("%q is not an operation of the ledger: taken or removed", op) + } + instance, err := InstanceOf(e.Dataset, e.Name) + if err != nil { + return err + } + if m.Ledger == nil { + return fmt.Errorf("this process reaches no ledger") + } + if op == "removed" { + return m.Ledger.Apply(instance, op, struct{}{}, call) + } + return m.Ledger.Apply(instance, op, e, call) +} diff --git a/modules/zfs/cmd/zfs-tools/snapshots_test.go b/modules/zfs/cmd/zfs-tools/snapshots_test.go new file mode 100644 index 000000000..b2aec5650 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshots_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +type ledgerCalls struct { + entries []LedgerEntry + read error + record error + records []map[string]any +} + +func (l *ledgerCalls) call(key string, body any) (json.RawMessage, error) { + switch key { + case ToolSchedule: + if l.read != nil { + return nil, l.read + } + return json.Marshal(map[string]any{"ledger": l.entries}) + case ToolRecord: + if l.record != nil { + return nil, l.record + } + l.records = append(l.records, body.(map[string]any)) + return json.RawMessage(`{}`), nil + } + return nil, errors.New("not a tool of the run: " + key) +} + +func snapFake(t *testing.T, keeps string, l *ledgerCalls) (*fake, *Machine, memStates) { + f := withZFS(t) + f.uid = 0 + f.now = time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local) // a Friday + m := f.machine() + st := memStates{} + m.States, m.Call = st, l.call + return f, m, st +} + +func keepsOfJSON(t *testing.T, s string) *Config { + c, ref := ParseConfig([]byte(`{"pool":"tank","snapshots":`+s+`}`), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + return c +} + +// §21, A-20, A-30, A-31, §31: a due kind takes one ZFS snapshot per filesystem of the pool (never a volume, +// never recursive), sets mesh:schedule, records taken, and says snapshot.taken.schedule; with the bus down, the +// ledger full, the pool above 80 % or the schedule paused none is taken, and snapshot.skipped.schedule says why. +func TestADueKindTakesOneSnapshotPerFilesystemAndRecordsEach(t *testing.T) { + stamp := time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local).UTC().Format(StampForm) + fresh := time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local).UTC().Format(time.RFC3339) + old := time.Date(2026, 10, 9, 10, 30, 0, 0, time.Local).UTC().Format(time.RFC3339) + cases := []struct { + name string + keeps string + l *ledgerCalls + setup func(f *fake, m *Machine, st memStates) + force bool + taken int + reason string + evKind string + wantErr bool + }{ + {name: "hourly due", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: []LedgerEntry{{"tank", "mesh-hourly-x", "hourly", old}}}, taken: 3, evKind: EventTakenSchedule}, + {name: "hourly already taken this hour", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: []LedgerEntry{{"tank", "mesh-hourly-x", "hourly", fresh}}}}, + {name: "weekly not on a Friday", keeps: `{"weekly":2}`, l: &ledgerCalls{}}, + {name: "keep none", keeps: `{"none":true}`, l: &ledgerCalls{}}, + {name: "a kind with 0 is not taken", keeps: `{"hourly":0,"daily":0}`, l: &ledgerCalls{}}, + {name: "bus down", keeps: `{"hourly":2}`, l: &ledgerCalls{read: errors.New("no bus")}, reason: "ledger-unavailable", evKind: EventSkippedSched}, + {name: "ledger full", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: make([]LedgerEntry, 6)}, reason: "ledger-full", evKind: EventSkippedSched}, + {name: "pool above 80 percent", keeps: `{"hourly":2}`, l: &ledgerCalls{}, reason: "full", evKind: EventSkippedSched, + setup: func(f *fake, _ *Machine, _ memStates) { + f.on(line("zpool", PoolsArgv()), "tank\t1000\t900\t100\t90\t1\tONLINE\t1.00\n") + }}, + {name: "paused", keeps: `{"hourly":2}`, l: &ledgerCalls{}, reason: "paused", evKind: EventSkippedSched, + setup: func(_ *fake, _ *Machine, st memStates) { st.Put(StateSchedule, KeyPaused, map[string]string{}) }}, + {name: "a person's act takes every kind and says .person", keeps: `{"hourly":1,"daily":1}`, l: &ledgerCalls{}, force: true, taken: 6, evKind: EventTakenPerson}, + {name: "the ledger refuses the record", keeps: `{"hourly":2}`, l: &ledgerCalls{record: errors.New("full")}, wantErr: true}, + } + for _, c := range cases { + f, m, st := snapFake(t, c.keeps, c.l) + if c.setup != nil { + c.setup(f, m, st) + } + cmds := map[string]bool{} + for _, ds := range []string{"tank", "tank/media", "tank/downloads"} { + for _, k := range []string{"hourly", "daily"} { + n := "mesh-" + k + "-" + stamp + f.on("zfs snapshot -- "+ds+"@"+n, "") + f.on("zfs set mesh:schedule="+k+" -- "+ds+"@"+n, "") + cmds[ds+"@"+n] = true + } + } + _, err := m.TakeScheduled(context.Background(), keepsOfJSON(t, c.keeps), c.force, "person@node") + if (err != nil) != c.wantErr { + t.Errorf("%s: error %v", c.name, err) + } + snaps := ran(f, "zfs snapshot") + if c.wantErr { + if len(snaps) == 0 { + t.Errorf("%s: nothing taken", c.name) + } + continue + } + if len(snaps) != c.taken || len(c.l.records) != c.taken || len(ran(f, "zfs set mesh:schedule=")) != c.taken { + t.Errorf("%s: %d snapshots, %d sets, %d records, want %d: %v", c.name, len(snaps), len(ran(f, "zfs set mesh:schedule=")), len(c.l.records), c.taken, f.calls) + } + for _, s := range snaps { + if strings.Contains(s, "tank/vm") || strings.Contains(s, " -r") { + t.Errorf("%s: %s", c.name, s) + } + } + if c.evKind == "" && len(f.emitted) != 0 || c.evKind != "" && (len(f.emitted) == 0 || f.emitted[0]["_type"] != c.evKind) { + t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.evKind) + } + if c.reason != "" && !strings.HasPrefix(f.emitted[0]["reason"].(string), c.reason) { + t.Errorf("%s: reason %v", c.name, f.emitted[0]["reason"]) + } + if c.force && f.emitted[0]["by"] != "person@node" { + t.Errorf("%s: %v", c.name, f.emitted[0]) + } + } +} + +// A-20: the tool zfs_snapshots_record refuses a snapshot not named mesh- and one of another operation. +type fakeLedger struct{ applied []string } + +func (l *fakeLedger) Apply(instance, op string, body any, call string) error { + l.applied = append(l.applied, op+" "+instance) + return nil +} +func (l *fakeLedger) Entries() ([]LedgerEntry, error) { return nil, nil } + +func TestTheLedgerTakesOnlyTheRunsSnapshots(t *testing.T) { + l := &fakeLedger{} + m := newFake(t).machine() + m.Ledger = l + e := LedgerEntry{Dataset: "tank/media", Name: "mesh-hourly-20261009T1000Z", Kind: "hourly", At: "2026-10-09T10:00:00Z"} + if err := m.Record("taken", e, "id"); err != nil || len(l.applied) != 1 || l.applied[0] != "taken snapshot/hourly-20261009T1000Z-tank~media" { + t.Fatalf("%v %v", err, l.applied) + } + if err := m.Record("removed", e, "id"); err != nil || l.applied[1] != "removed snapshot/hourly-20261009T1000Z-tank~media" { + t.Fatal(err, l.applied) + } + for _, bad := range []LedgerEntry{{Dataset: "tank", Name: "before-move"}, {Dataset: "tank", Name: "mesh-yearly-20261009T1000Z"}, + {Dataset: "", Name: "mesh-hourly-20261009T1000Z"}, {Dataset: "tank@x", Name: "mesh-hourly-20261009T1000Z"}} { + if err := m.Record("taken", bad, ""); err == nil { + t.Errorf("accepted %+v", bad) + } + } + if err := m.Record("destroy", e, ""); err == nil || len(l.applied) != 2 { + t.Errorf("another operation: %v %v", err, l.applied) + } +} diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go index d77989000..9d4ff7f44 100644 --- a/modules/zfs/cmd/zfs-tools/tools.go +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -86,6 +86,26 @@ func Tools(m *Machine) []stdio.Tool { return map[string]any{"paused": false}, nil }, }, + { + Name: "zfs_snapshots_record", + Description: "Record the scheduled run's `taken` or `removed` ZFS snapshot in the ledger `snapshots`: the run is a process, " + + "which cannot write a ledger, so it calls this tool. Refused for a snapshot not named mesh--: the ledger " + + "holds only the run's. (a)", + Input: map[string]any{ + "operation": map[string]any{"type": "string", "description": "taken or removed"}, + "dataset": map[string]any{"type": "string", "description": "the dataset"}, + "name": map[string]any{"type": "string", "description": "the snapshot's name after the @"}, + "kind": map[string]any{"type": "string", "description": "hourly, daily, weekly or monthly"}, + "at": map[string]any{"type": "string", "description": "when it was taken, RFC 3339 in UTC"}, + }, + Run: func(a map[string]any) (any, error) { + e := LedgerEntry{Dataset: text(a, "dataset"), Name: text(a, "name"), Kind: text(a, "kind"), At: text(a, "at")} + if err := m.Record(text(a, "operation"), e, stdio.CallIDOf(a)); err != nil { + return nil, err + } + return map[string]any{"recorded": true}, nil + }, + }, { Name: "zfs_pools", Description: "Every imported pool's health in a line: its state (ONLINE, DEGRADED, FAULTED, …) with status's own " + diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 082c8dcbe..431e2fb24 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -64,6 +64,7 @@ "zfs_config_apply", "zfs_snapshots_pause", "zfs_snapshots_resume", + "zfs_snapshots_record", "zfs_io", "zfs_arc", "zfs_events", From 23256413de6cd0006f8371b732fbbfa699c842ad Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:44:07 +0100 Subject: [PATCH 10/35] Z12 zfs-snapshots-remove: the run removes surplus scheduled ZFS snapshots only by their own record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §22, §26, A-30, A-31 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 3 +- modules/zfs/cmd/zfs-tools/run.go | 11 ++ modules/zfs/cmd/zfs-tools/snapshots_prune.go | 156 ++++++++++++++++++ .../zfs/cmd/zfs-tools/snapshots_prune_test.go | 121 ++++++++++++++ 4 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 modules/zfs/cmd/zfs-tools/snapshots_prune.go create mode 100644 modules/zfs/cmd/zfs-tools/snapshots_prune_test.go diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 45c15b9fb..35e20bf7e 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -228,7 +228,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, "DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"}, "AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"}, - "SnapshotSetArgv": {"zfs", "set"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, + "SnapshotSetArgv": {"zfs", "set"}, "HoldsArgv": {"zfs", "holds"}, "SnapshotPropertyArgv": {"zfs", "get"}, "DestroyArgv": {"zfs", "destroy"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), @@ -240,6 +240,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DeviceStatusArgv": DeviceStatusArgv("tank", true), "AllRealStatusArgv": AllRealStatusArgv(), "RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"), "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x"), + "HoldsArgv": HoldsArgv("tank@x"), "SnapshotPropertyArgv": SnapshotPropertyArgv("tank@x", "clones"), "DestroyArgv": DestroyArgv("tank", "x"), "SnapshotSetArgv": SnapshotSetArgv("tank@mesh-hourly-x", "hourly"), "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")} for fn, argv := range verbs { for _, a := range argv { diff --git a/modules/zfs/cmd/zfs-tools/run.go b/modules/zfs/cmd/zfs-tools/run.go index a99894446..a6bd87aed 100644 --- a/modules/zfs/cmd/zfs-tools/run.go +++ b/modules/zfs/cmd/zfs-tools/run.go @@ -112,6 +112,12 @@ func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, if err != nil { failed = append(failed, err) } + case act == ActPrune && guard == "": + ts, err := m.PruneScheduled(ctx, c, true, by) + said = append(said, ts...) + if err != nil { + failed = append(failed, err) + } case guard != "": said = append(said, m.skip(guard)) default: @@ -163,6 +169,11 @@ func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, if err != nil { failed = append(failed, err) } + ps, err := m.PruneScheduled(ctx, c, false, "") + said = append(said, ps...) + if err != nil { + failed = append(failed, err) + } if len(said) == 0 { said = append(said, "nothing due") } diff --git a/modules/zfs/cmd/zfs-tools/snapshots_prune.go b/modules/zfs/cmd/zfs-tools/snapshots_prune.go new file mode 100644 index 000000000..ab2f9dc47 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshots_prune.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" +) + +// The run's ZFS snapshots: removing them, only by their own record (design 42 zfs-owns-zfs §22). +// +// After taking, the run reads the ledger again in the same run (no read, no removal) and, per dataset and kind, +// removes the oldest past its keep count, all of them in that run. Before any destroy it checks that the name +// is `@mesh--`, that the snapshot carries `mesh:schedule=`, that +// it is in the ledger, that it has no hold and no clone; anything else is refused, said and left. Then `zfs +// destroy -- @`, the ledger's `removed`, and `snapshot.removed.schedule`. When `snapshots` +// goes back to `none` nothing is removed. A ZFS snapshot taken by hand is never removed. + +// HoldsArgv lists a snapshot's holds. +func HoldsArgv(snapshot string) []string { return []string{"holds", "-H", "--", snapshot} } + +// SnapshotPropertyArgv reads one property of a snapshot (mesh:schedule, clones). +func SnapshotPropertyArgv(snapshot, property string) []string { + return []string{"get", "-Hp", "-o", "value", property, "--", snapshot} +} + +// DestroyArgv destroys one snapshot, never a dataset: the name always holds its @. +func DestroyArgv(dataset, snapshot string) []string { + return []string{"destroy", "--", dataset + "@" + snapshot} +} + +// surplus are the entries of a dataset and kind past its keep count, oldest first. +func surplus(entries []LedgerEntry, keep int) []LedgerEntry { + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].At != entries[j].At { + return entries[i].At < entries[j].At + } + return entries[i].Name < entries[j].Name + }) + if len(entries) <= keep { + return nil + } + return entries[:len(entries)-keep] +} + +// refuseRemoval is the reason an entry may not be destroyed, "" when every check holds. +func (m *Machine) refuseRemoval(ctx context.Context, pool string, e LedgerEntry) (string, error) { + full := e.Dataset + "@" + e.Name + nm := meshSnapshot.FindStringSubmatch(e.Name) + switch { + case e.Dataset != pool && !strings.HasPrefix(e.Dataset, pool+"/"): + return "its dataset is not a dataset of the pool", nil + case nm == nil || nm[1] != e.Kind: + return "its name is not mesh-" + e.Kind + "-", nil + } + sched, err := m.ask(ctx, "zfs", SnapshotPropertyArgv(full, "mesh:schedule")...) + if err != nil { + return "", err + } + if strings.TrimSpace(sched) != e.Kind { + return fmt.Sprintf("it carries mesh:schedule=%q, not %q", strings.TrimSpace(sched), e.Kind), nil + } + holds, err := m.ask(ctx, "zfs", HoldsArgv(full)...) + if err != nil { + return "", err + } + if strings.TrimSpace(holds) != "" { + return "it has a hold", nil + } + clones, err := m.ask(ctx, "zfs", SnapshotPropertyArgv(full, "clones")...) + if err != nil { + return "", err + } + if c := strings.TrimSpace(clones); c != "" && c != "-" { + return "it has a clone", nil + } + return "", nil +} + +// PruneScheduled removes the scheduled ZFS snapshots past their keep counts (a person's act when person, by +// them). It answers what it did, and a failed command as the error. +func (m *Machine) PruneScheduled(ctx context.Context, c *Config, person bool, by string) ([]string, error) { + if c.Snapshots == nil || c.Snapshots.None || c.Pool == "" { + return nil, nil + } + removed, skipped := EventRemovedSched, EventSkippedSched + if person { + removed, skipped = EventRemovedPerson, EventSkippedPerson + } + if m.Paused() { + return []string{"snapshots paused: nothing removed"}, nil + } + entries, err := m.ledgerEntries() + if err != nil { + return []string{"nothing removed: the ledger cannot be read"}, nil + } + snaps, err := m.Snapshots(ctx) + if err != nil { + return nil, err + } + have := map[string]Snapshot{} + for _, s := range snaps { + have[s.Name] = s + } + groups := map[string][]LedgerEntry{} + for _, e := range entries { + k := e.Dataset + " " + e.Kind + groups[k] = append(groups[k], e) + } + var said []string + var failed []error + capacity := 0 + if ps, err := m.Pools(ctx); err == nil { + for _, p := range ps { + if p.Name == c.Pool { + capacity = p.Capacity + } + } + } + for _, k := range sortedKeys(groups) { + g := groups[k] + for _, e := range surplus(g, c.Snapshots.Of(g[0].Kind)) { + full := e.Dataset + "@" + e.Name + why, err := m.refuseRemoval(ctx, c.Pool, e) + if err != nil { + failed = append(failed, fmt.Errorf("%s: %v", full, err)) + continue + } + if why != "" { + var f2 []error + if err := m.say(skipped, map[string]any{"pool": c.Pool, "capacity": capacity, "reason": "refused to remove " + full + ": " + why}); err != nil { + m.failure(&f2, skipped, err) + } + said = append(said, "left "+full+": "+why) + continue + } + freed := have[full].Used + if _, there := have[full]; there { + if _, err := m.AsRoot(ctx, "zfs", DestroyArgv(e.Dataset, e.Name)...); err != nil { + failed = append(failed, fmt.Errorf("%s: %v", full, err)) + continue + } + } + if err := m.recordLedger("removed", e); err != nil { + failed = append(failed, fmt.Errorf("%s was removed, and the ledger did not take it: %v", full, err)) + continue + } + if err := m.say(removed, map[string]any{"snapshot": full, "kind": e.Kind, "space freed in bytes": freed, "by": by}); err != nil { + m.failure(&failed, removed, err) + } + said = append(said, fmt.Sprintf("removed %s (%s freed)", full, strconv.FormatUint(freed, 10))) + } + } + return said, joinErrors(failed) +} diff --git a/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go b/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go new file mode 100644 index 000000000..d1ef491a7 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" +) + +// §22, §31: pruning removes every surplus of a dataset and kind in one run, oldest first, only by the +// ledger's record, and never a snapshot whose name, property or ledger entry does not fit, nor one with a hold +// or a clone; `none` and a pause remove nothing; a hand-made snapshot is never touched. +func TestPruningRemovesEverySurplusByItsRecordAndNothingThatDoesNotFit(t *testing.T) { + name := func(h int) string { return fmt.Sprintf("mesh-hourly-20261009T%02d00Z", h) } + at := func(h int) string { return fmt.Sprintf("2026-10-09T%02d:00:00Z", h) } + entry := func(ds string, h int) LedgerEntry { return LedgerEntry{ds, name(h), "hourly", at(h)} } + four := []LedgerEntry{entry("tank", 1), entry("tank", 2), entry("tank", 3), entry("tank", 4)} + cases := []struct { + name string + keeps string + l *ledgerCalls + prop map[string]string // by snapshot: its mesh:schedule + hold string // the snapshot with a hold + clone string // the snapshot with a clone + gone string // the snapshot ZFS no longer has + setup func(st memStates) + removed []string + left int + wantErr bool + }{ + {name: "every surplus in one run, oldest first", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: four}, + removed: []string{"tank@" + name(1), "tank@" + name(2)}}, + {name: "within the keep count", keeps: `{"hourly":4}`, l: &ledgerCalls{entries: four}}, + {name: "a hold is left", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: four}, hold: "tank@" + name(1), + removed: []string{"tank@" + name(2)}, left: 1}, + {name: "a clone is left", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: four}, clone: "tank@" + name(2), + removed: []string{"tank@" + name(1)}, left: 1}, + {name: "a property that does not fit is left", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: four}, + prop: map[string]string{"tank@" + name(1): "daily"}, removed: []string{"tank@" + name(2)}, left: 1}, + {name: "a dataset of another pool is left", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: []LedgerEntry{entry("other", 1), entry("other", 2)}}, left: 1}, + {name: "a name that does not fit is left", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: []LedgerEntry{ + {"tank", "before-move", "hourly", at(1)}, entry("tank", 2)}}, left: 1}, + {name: "none removes nothing", keeps: `{"none":true}`, l: &ledgerCalls{entries: four}}, + {name: "paused removes nothing", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: four}, + setup: func(st memStates) { st.Put(StateSchedule, KeyPaused, map[string]string{}) }}, + {name: "no read, no removal", keeps: `{"hourly":1}`, l: &ledgerCalls{read: errors.New("no bus")}}, + {name: "a snapshot ZFS no longer has is only recorded removed", keeps: `{"hourly":3}`, l: &ledgerCalls{entries: four}, + gone: "tank@" + name(1)}, + {name: "a failed destroy is the run's failure", keeps: `{"hourly":3}`, l: &ledgerCalls{entries: four}, wantErr: true}, + } + for _, c := range cases { + f, m, st := snapFake(t, c.keeps, c.l) + if c.setup != nil { + c.setup(st) + } + var list strings.Builder + for _, e := range c.l.entries { + full := e.Dataset + "@" + e.Name + if full != c.gone { + fmt.Fprintf(&list, "%s\t1791400000\t1000\t2000\n", full) + } + p := c.prop[full] + if p == "" { + p = e.Kind + } + f.on("zfs get -Hp -o value mesh:schedule -- "+full, p+"\n") + h := "" + if full == c.hold { + h = full + "\tkeep\tFri Oct 9 12:00 2026\n" + } + f.on("zfs holds -H -- "+full, h) + cl := "-" + if full == c.clone { + cl = "tank/clone" + } + f.on("zfs get -Hp -o value clones -- "+full, cl+"\n") + if !c.wantErr { + f.on("zfs destroy -- "+full, "") + } + } + f.on(line("zfs", SnapshotsArgv()), list.String()+"tank@by-hand\t1791400000\t1\t2\n") + said, err := m.PruneScheduled(context.Background(), keepsOfJSON(t, c.keeps), false, "") + if (err != nil) != c.wantErr { + t.Errorf("%s: error %v", c.name, err) + } + var destroyed []string + for _, d := range ran(f, "zfs destroy") { + destroyed = append(destroyed, strings.TrimPrefix(d, "zfs destroy -- ")) + } + if c.gone != "" { + if len(destroyed) != 0 || len(c.l.records) != 1 || c.l.records[0]["operation"] != "removed" { + t.Errorf("%s: destroyed %v, records %v", c.name, destroyed, c.l.records) + } + continue + } + if strings.Join(destroyed, ",") != strings.Join(c.removed, ",") && !c.wantErr { + t.Errorf("%s: destroyed %v, want %v (%v)", c.name, destroyed, c.removed, said) + } + if !c.wantErr && len(c.l.records) != len(c.removed) { + t.Errorf("%s: %d records for %d removals", c.name, len(c.l.records), len(c.removed)) + } + skips := 0 + for _, e := range f.emitted { + if e["_type"] == EventSkippedSched { + skips++ + } + if e["_type"] == EventRemovedSched && e["space freed in bytes"] != uint64(1000) { + t.Errorf("%s: freed %v", c.name, e) + } + } + if skips != c.left || len(f.emitted)-skips != len(c.removed) { + t.Errorf("%s: events %v, left %d", c.name, f.emitted, c.left) + } + for _, d := range destroyed { + if !strings.Contains(d, "@mesh-") || strings.Contains(d, "by-hand") { + t.Errorf("%s: destroyed %s", c.name, d) + } + } + } +} From dd1a4e083c52692c8f9adeff603614aa5e8d12c7 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:49:01 +0100 Subject: [PATCH 11/35] Z14 zfs-follower: the follower starts the apply on the events that end a wait and keeps its findings in its own state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §18, §27, A-10, A-16, A-32 --- modules/zfs/cmd/zfs-tools/bus.go | 2 + modules/zfs/cmd/zfs-tools/follow.go | 24 +++ modules/zfs/cmd/zfs-tools/follow_wake.go | 198 ++++++++++++++++++ modules/zfs/cmd/zfs-tools/follow_wake_test.go | 93 ++++++++ modules/zfs/cmd/zfs-tools/machine.go | 3 + modules/zfs/cmd/zfs-tools/manifest_test.go | 6 +- modules/zfs/go.mod | 5 + 7 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/follow_wake.go create mode 100644 modules/zfs/cmd/zfs-tools/follow_wake_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 41ec813b3..c846628ba 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/process" @@ -48,6 +49,7 @@ func (m *Machine) JoinBus() func() { } m.Emit = func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) } m.States = busStates{c} + m.Findings = meshconditionsclient.FindingWriter{State: c.State(StateFindings)} m.Call = c.Call return c.Close } diff --git a/modules/zfs/cmd/zfs-tools/follow.go b/modules/zfs/cmd/zfs-tools/follow.go index 495316ee4..0eea6d2e6 100644 --- a/modules/zfs/cmd/zfs-tools/follow.go +++ b/modules/zfs/cmd/zfs-tools/follow.go @@ -58,6 +58,8 @@ type Cursor struct { EID uint64 `json:"eid"` Time time.Time `json:"time"` path string + // OnEvent hears every event not read before, after the cursor passed it. + OnEvent func(Event) `json:"-"` } // LoadCursor is the cursor kept for this boot; an empty one for another boot or none kept. @@ -149,6 +151,9 @@ func Follow(r io.Reader, out io.Writer, cur *Cursor) error { cur.Time = at } _ = cur.Save(cur.path) + if cur.OnEvent != nil { + cur.OnEvent(e) + } } block = block[:0] } @@ -260,6 +265,25 @@ func StartZpoolFollow(ctx context.Context) (FollowProcess, error) { func Follower(ctx context.Context, out io.Writer) error { boot, _ := os.ReadFile(BootID) cur := LoadCursor(CursorFile, strings.TrimSpace(string(boot))) + m := Here() + defer m.JoinBus()() + warn := func(what string, err error) { + if err != nil { + fmt.Fprintf(out, "%s: %v\n", what, err) + } + } + // Once at its own start (A-16), then on the events that end a wait. + warn("starting the apply", m.StartApply()) + warn("reading the findings", m.SyncFindings(ctx)) + cur.OnEvent = func(e Event) { + if EndsAWait(e.Class) { + warn("starting the apply", m.StartApply()) + } + if ChangesAFinding(e.Class) { + warn("reading the findings", m.SyncFindings(ctx)) + } + } + go func() { warn("following udev", m.StartUdev(ctx)) }() return RunFollower(ctx, out, StartZpoolFollow, cur, CursorFile, func(d time.Duration) { select { case <-ctx.Done(): diff --git a/modules/zfs/cmd/zfs-tools/follow_wake.go b/modules/zfs/cmd/zfs-tools/follow_wake.go new file mode 100644 index 000000000..51a8529ca --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/follow_wake.go @@ -0,0 +1,198 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "os/exec" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// What the event follower does besides the journal (design 42 zfs-owns-zfs §18, §27; A-10, A-16). +// +// **A wait is ended by an event.** The follower, which runs from boot, starts the apply unit once at its own +// start and when it hears a scrub or resilver finish (`scrub_finish`, `resilver_finish`) or a declared device +// appear (a udev `add` of a block device whose link under /dev/disk/by-id is declared), through +// `/node-service-manager.start`. It also keeps the findings: one key `pool..` per device of +// the pool, `pool..pool` for the pool, not ONLINE, its value the state, deleted when the device is ONLINE +// again or leaves the pool. + +// ApplyUnit is the unit of the apply process (`zfs-config-apply`). +const ApplyUnit = "zfs-config-apply.service" + +// UdevArgv follows the kernel's block device events. +func UdevArgv() []string { return []string{"monitor", "--udev", "--subsystem-match=block"} } + +// StartApply starts the apply unit through the service manager. +func (m *Machine) StartApply() error { + if m.Call == nil { + return fmt.Errorf("this process reaches no seat, so the apply was not started") + } + _, err := m.Call("seat:node-service-manager.start@"+m.Node, map[string]any{"scope": "system", "unit": ApplyUnit}) + return err +} + +// EndsAWait says whether an event of ZFS's ends a wait of the apply. +func EndsAWait(class string) bool { + return strings.HasSuffix(class, ".scrub_finish") || strings.HasSuffix(class, ".resilver_finish") +} + +// ChangesAFinding says whether an event of ZFS's may change a device's state. +func ChangesAFinding(class string) bool { + return strings.HasSuffix(class, ".statechange") || strings.HasPrefix(class, "resource.fs.zfs.") || + strings.HasSuffix(class, ".config_sync") || EndsAWait(class) +} + +// DeclaredAdded says whether a line of `udevadm monitor` is the add of a block device a declared name links to. +// The declared names are read from the config file at each add, the links resolved now. +func (m *Machine) DeclaredAdded(line string) bool { + f := strings.Fields(line) + if len(f) < 4 || f[0] != "UDEV" || f[2] != "add" { + return false + } + added := filepath.Base(f[3]) + text, err := m.ReadFile(ConfigFile) + if err != nil { + return false + } + c, refused := ParseConfig(text, m.memMiB()) + if len(refused) > 0 { + return false + } + for _, d := range append(m.resolveDeclared(c.CacheDevices, "cache"), m.resolveDeclared(c.LogDevices, "log")...) { + if d.Present && partOf(added, filepath.Base(d.Real)) { + return true + } + } + return false +} + +// FollowUdev reads `udevadm monitor` and starts the apply on the add of a declared device. +func (m *Machine) FollowUdev(r io.Reader) error { + sc := bufio.NewScanner(r) + for sc.Scan() { + if m.DeclaredAdded(sc.Text()) { + if err := m.StartApply(); err != nil { + return err + } + } + } + return sc.Err() +} + +// StartUdev starts `udevadm monitor` and follows it until the context ends. +func (m *Machine) StartUdev(ctx context.Context) error { + cmd := exec.Command("udevadm", UdevArgv()...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + out, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + go func() { + <-ctx.Done() + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + }() + ferr := m.FollowUdev(out) + _ = cmd.Wait() + return ferr +} + +// FindingsOf are the findings `zpool status -P` shows now: key to state, for a pool or a device not ONLINE. +func FindingsOf(out string) (map[string]string, error) { + ps, err := ParseStatus(out) + if err != nil { + return nil, err + } + found := map[string]string{} + var walk func(pool string, d *Device) + walk = func(pool string, d *Device) { + for _, c := range d.Children { + if c.Kind != "section" && c.State != "" && c.State != "ONLINE" && c.State != "AVAIL" && c.State != "INUSE" { + found["pool."+pool+"."+c.Name] = c.State + } + walk(pool, c) + } + } + for _, p := range ps { + if p.State != "ONLINE" { + found["pool."+p.Name+".pool"] = p.State + } + if p.Root != nil { + walk(p.Name, p.Root) + } + } + return found, nil +} + +// FindingsObserver is what the findings are written as: the module is their observer, as every observer on the mesh +// writes its own state `findings` with mesh-sdk's views.Finding and the seat mesh-conditions' writer (A-32). +const FindingsObserver = "zfs" + +// findingOf is the finding of one device or pool not ONLINE: the module is its subject, the pool and device its +// kind, `pool..`, so each is a key of its own and goes when the device is ONLINE again or gone. +func findingOf(kind, state string, at time.Time) views.Finding { + name := strings.TrimPrefix(kind, "pool.") + return views.Finding{Observer: FindingsObserver, Kind: kind, ID: FindingsObserver, + Summary: fmt.Sprintf("%s is %s", name, state), + Headline: fmt.Sprintf("ZFS %s is %s", name, state), + Explanation: "ZFS reports this pool or device not ONLINE; zfs_pool_layout and zfs_pools say more. The finding clears when it is ONLINE again or leaves the pool.", + At: at} +} + +// SyncFindings reads `zpool status -P` against the keys of the state `findings`, writes what is new or changed, +// and clears what is gone, each through the writer. +func (m *Machine) SyncFindings(ctx context.Context) error { + if m.States == nil || m.Findings == nil { + return fmt.Errorf("this process keeps no state, so the findings were not read") + } + out, err := m.ask(ctx, "zpool", StatusPathsArgv()...) + if err != nil { + return err + } + now, err := FindingsOf(out) + if err != nil { + return err + } + keys, err := m.States.Keys(StateFindings) + if err != nil { + return err + } + prefix := strings.TrimSuffix(meshconditionsclient.Key(meshconditionsclient.ScopeModule, FindingsObserver, "x"), "x") + wanted := map[string]bool{} + for _, kind := range sortedKeys(now) { + f := findingOf(kind, now[kind], m.Now()) + key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, FindingsObserver, kind) + wanted[key] = true + if raw, ok, _ := m.States.Get(StateFindings, key); ok && strings.Contains(string(raw), `"summary":"`+f.Summary+`"`) { + continue + } + if err := m.Findings.Put(f); err != nil { + return err + } + } + sort.Strings(keys) + for _, k := range keys { + kind, mine := strings.CutPrefix(k, prefix) + if !mine || wanted[k] { + continue + } + if err := m.Findings.Delete(views.Finding{ID: FindingsObserver, Kind: kind}); err != nil { + return err + } + } + return nil +} + +// StatusPathsArgv reads every pool's status with the paths its devices are held by. +func StatusPathsArgv() []string { return []string{"status", "-P"} } diff --git a/modules/zfs/cmd/zfs-tools/follow_wake_test.go b/modules/zfs/cmd/zfs-tools/follow_wake_test.go new file mode 100644 index 000000000..ac65229fb --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/follow_wake_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-sdk/go/views" +) + +type findingWriter struct { + st memStates + puts []string + dels []string +} + +func (w *findingWriter) Put(f views.Finding) error { + key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, f.ID, f.Kind) + w.puts = append(w.puts, key) + return w.st.Put(StateFindings, key, map[string]string{"summary": f.Summary}) +} +func (w *findingWriter) Delete(f views.Finding) error { + key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, f.ID, f.Kind) + w.dels = append(w.dels, key) + return w.st.Delete(StateFindings, key) +} + +// §18, §27, A-16: the apply is started once at the follower's start and on scrub_finish, resilver_finish and a +// declared device's udev add, through the service manager; a device not ONLINE is a finding, written once, +// deleted when ONLINE again or gone. +func TestTheFollowerStartsTheApplyOnTheEventsThatEndAWaitAndKeepsTheFindings(t *testing.T) { + for class, want := range map[string]bool{"sysevent.fs.zfs.scrub_finish": true, "sysevent.fs.zfs.resilver_finish": true, + "sysevent.fs.zfs.scrub_start": false, "ereport.fs.zfs.io": false} { + if EndsAWait(class) != want { + t.Errorf("%s: %v", class, !want) + } + } + f := newFake(t) + f.uid = 0 + f.files[ConfigFile] = `{"pool":"tank","cache-devices":["c1"]}` + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + f.links["/dev/disk/by-id/c1"] = "/dev/nvme1n1" + m := f.machine() + var key string + var body any + m.Node = "ace" + m.Call = func(k string, b any) (json.RawMessage, error) { key, body = k, b; return nil, nil } + udev := map[string]bool{ + "UDEV [12.3] add /devices/pci0000:00/nvme/nvme1/nvme1n1 (block)": true, + "UDEV [12.3] add /devices/pci0000:00/nvme/nvme1/nvme1n1/nvme1n1p1 (block)": true, + "UDEV [12.3] add /devices/pci0000:00/nvme/nvme2/nvme2n1 (block)": false, + "UDEV [12.3] remove /devices/pci0000:00/nvme/nvme1/nvme1n1 (block)": false, + "KERNEL[12.3] add /devices/pci0000:00/nvme/nvme1/nvme1n1 (block)": false, + } + for line, want := range udev { + if m.DeclaredAdded(line) != want { + t.Errorf("%q: %v", line, !want) + } + } + if err := m.FollowUdev(strings.NewReader("UDEV [1] add /devices/x/nvme2n1 (block)\nUDEV [2] add /devices/x/nvme1n1 (block)\n")); err != nil { + t.Fatal(err) + } + args, _ := body.(map[string]any) + if key != "seat:node-service-manager.start@ace" || args["unit"] != ApplyUnit || args["scope"] != "system" { + t.Errorf("called %s with %v", key, body) + } + + // The findings: a degraded pool, then healthy again. + st := memStates{} + w := &findingWriter{st: st} + m.States, m.Findings = st, w + f.on(line("zpool", StatusPathsArgv()), fixture(t, "zpool-status-degraded.txt")) + if err := m.SyncFindings(context.Background()); err != nil { + t.Fatal(err) + } + for _, k := range []string{"pool_tank_pool", "pool_tank_raidz2-0", "pool_tank_ata-WDC_WD120EFBX-68B0EN0_D7JA0002"} { + if _, ok := st["findings/module.zfs."+k]; !ok { + t.Errorf("no finding %s in %v", k, st) + } + } + if len(w.puts) != 4 { + t.Errorf("puts %v", w.puts) + } + if err := m.SyncFindings(context.Background()); err != nil || len(w.puts) != 4 { + t.Errorf("an unchanged finding was written again: %v %v", err, w.puts) + } + f.on(line("zpool", StatusPathsArgv()), fixture(t, "zpool-status.txt")) + if err := m.SyncFindings(context.Background()); err != nil || len(w.dels) != 4 || len(st) != 0 { + t.Errorf("deleted %v, left %v, %v", w.dels, st, err) + } +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 669aa5039..ee817c47d 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/views" "os" "os/exec" "path/filepath" @@ -127,6 +128,8 @@ type Machine struct { // Await waits at most the bound for an event of the module whose kind is one of kinds, and answers it; false // when none came. Await func(kinds []string, bound time.Duration) (map[string]any, bool) + // Findings writes the module's own state `findings` as the seat mesh-conditions' layout says (A-32). + Findings views.FindingWriter // Warnings are what the apply saw that is no failure (a change the bus did not take, a parameter awaiting // a load). Warnings []string diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 35e20bf7e..c7de1e430 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -228,7 +228,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, "DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"}, "AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"}, - "SnapshotSetArgv": {"zfs", "set"}, "HoldsArgv": {"zfs", "holds"}, "SnapshotPropertyArgv": {"zfs", "get"}, "DestroyArgv": {"zfs", "destroy"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, + "UdevArgv": {"udevadm", "monitor"}, "SnapshotSetArgv": {"zfs", "set"}, "StatusPathsArgv": {"zpool", "status"}, "HoldsArgv": {"zfs", "holds"}, "SnapshotPropertyArgv": {"zfs", "get"}, "DestroyArgv": {"zfs", "destroy"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"}, } verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), @@ -241,6 +241,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"), "WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x"), "HoldsArgv": HoldsArgv("tank@x"), "SnapshotPropertyArgv": SnapshotPropertyArgv("tank@x", "clones"), "DestroyArgv": DestroyArgv("tank", "x"), + "StatusPathsArgv": StatusPathsArgv(), "UdevArgv": UdevArgv(), "SnapshotSetArgv": SnapshotSetArgv("tank@mesh-hourly-x", "hourly"), "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")} for fn, argv := range verbs { for _, a := range argv { @@ -262,7 +263,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { } } call := regexp.MustCompile(`(\.(ask|AsRoot|Escalated|Run|Start|Output|CombinedOutput)\(|exec\.Command(Context)?\(|os\.StartProcess\(|syscall\.(Exec|ForkExec)\()`) - site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb|zgenhostid)", (\w+)\(`) + site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb|zgenhostid|udevadm)", (\w+)\(`) // The bodies that run what they are given, each once. bodies := map[string]bool{ "machine.go: r := m.Run(ctx, name, args...)": true, @@ -271,6 +272,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "machine.go: cmd := exec.Command(name, args...)": true, "machine.go: if err := cmd.Start(); err != nil {": true, "follow.go: if err := cmd.Start(); err != nil {": true, + "follow_wake.go: if err := cmd.Start(); err != nil {": true, } files, _ := filepath.Glob("*.go") sites := 0 diff --git a/modules/zfs/go.mod b/modules/zfs/go.mod index 7b7046b1b..804a627cf 100644 --- a/modules/zfs/go.mod +++ b/modules/zfs/go.mod @@ -4,6 +4,11 @@ go 1.26.0 require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 +require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect +) + require ( github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nats.go v1.54.0 // indirect From 8aad282e6c4c6a4a2730c3c8a7decdb49d37f52f Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:51:11 +0100 Subject: [PATCH 12/35] Z10 Z12 trim without day or time refused, keep count 0 removes nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trim schedule of weekly or monthly with no trim-day or trim-time is one refused value naming it; a keep count of 0 is that kind's none: the existing scheduled ZFS snapshots are kept and a warning names them. Builds: design 42 zfs-owns-zfs §20, §21, §22, A-33, A-35 --- modules/zfs/cmd/zfs-tools/config.go | 12 ++++++++++++ modules/zfs/cmd/zfs-tools/config_test.go | 2 ++ modules/zfs/cmd/zfs-tools/snapshots_prune.go | 10 ++++++++++ modules/zfs/cmd/zfs-tools/snapshots_prune_test.go | 6 ++++++ 4 files changed, 30 insertions(+) diff --git a/modules/zfs/cmd/zfs-tools/config.go b/modules/zfs/cmd/zfs-tools/config.go index b608a09cd..54462242c 100644 --- a/modules/zfs/cmd/zfs-tools/config.go +++ b/modules/zfs/cmd/zfs-tools/config.go @@ -263,6 +263,18 @@ func ParseConfig(text []byte, memMiB uint64) (*Config, []Refusal) { return oneOf(s, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") }) c.TrimTime, _ = p.str("trim-time", clockForm) + // A-33: a trim schedule of weekly or monthly with no day or no time runs no trim and is said, as one refused value. + if c.TrimSchedule == "weekly" || c.TrimSchedule == "monthly" { + var missing []string + for _, need := range []struct{ key, have string }{{"trim-day", c.TrimDay}, {"trim-time", c.TrimTime}} { + if need.have == "" { + missing = append(missing, need.key) + } + } + if len(missing) > 0 { + p.refuse("trim-schedule", p.raw["trim-schedule"], "needs "+strings.Join(missing, " and ")+" set: without it no trim runs") + } + } c.Zed, _ = p.str("zed", func(s string) string { return oneOf(s, "on", "off") }) if h, ok := p.str("hostid", func(s string) string { switch { diff --git a/modules/zfs/cmd/zfs-tools/config_test.go b/modules/zfs/cmd/zfs-tools/config_test.go index af1298096..52b498129 100644 --- a/modules/zfs/cmd/zfs-tools/config_test.go +++ b/modules/zfs/cmd/zfs-tools/config_test.go @@ -31,6 +31,8 @@ func TestEachValueHasItsFormAndABlankOneIsNotManaged(t *testing.T) { {"trim schedule", bad("trim-schedule", `"daily"`), "none, weekly, monthly"}, {"trim day", bad("trim-day", `"Monday"`), "Mon"}, {"trim time", bad("trim-time", `"24:00"`), "HH:MM"}, + {"trim schedule without day or time", `{"trim-schedule":"weekly","trim-time":"03:00"}`, "needs trim-day"}, + {"trim schedule without time", `{"trim-schedule":"monthly","trim-day":"Sun"}`, "needs trim-time"}, {"cache device with a slash", bad("cache-devices", `["a/b"]`), "under /dev/disk/by-id"}, {"cache device with a dollar", bad("cache-devices", `["a$b"]`), "under /dev/disk/by-id"}, {"cache device with a quote", bad("cache-devices", `["a\"b"]`), "under /dev/disk/by-id"}, diff --git a/modules/zfs/cmd/zfs-tools/snapshots_prune.go b/modules/zfs/cmd/zfs-tools/snapshots_prune.go index ab2f9dc47..e053a8234 100644 --- a/modules/zfs/cmd/zfs-tools/snapshots_prune.go +++ b/modules/zfs/cmd/zfs-tools/snapshots_prune.go @@ -120,6 +120,16 @@ func (m *Machine) PruneScheduled(ctx context.Context, c *Config, person bool, by } for _, k := range sortedKeys(groups) { g := groups[k] + if c.Snapshots.Of(g[0].Kind) == 0 { + // A-35: a keep count of 0 is that kind's none: its scheduled snapshots are kept and a warning names them. + names := make([]string, 0, len(g)) + for _, e := range g { + names = append(names, e.Dataset+"@"+e.Name) + } + sort.Strings(names) + m.Warnings = append(m.Warnings, fmt.Sprintf("the keep count of %s is 0, which removes nothing: kept %s", g[0].Kind, strings.Join(names, ", "))) + continue + } for _, e := range surplus(g, c.Snapshots.Of(g[0].Kind)) { full := e.Dataset + "@" + e.Name why, err := m.refuseRemoval(ctx, c.Pool, e) diff --git a/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go b/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go index d1ef491a7..d09388b66 100644 --- a/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go +++ b/modules/zfs/cmd/zfs-tools/snapshots_prune_test.go @@ -27,6 +27,7 @@ func TestPruningRemovesEverySurplusByItsRecordAndNothingThatDoesNotFit(t *testin setup func(st memStates) removed []string left int + warns string // a warning the run must have kept wantErr bool }{ {name: "every surplus in one run, oldest first", keeps: `{"hourly":2}`, l: &ledgerCalls{entries: four}, @@ -42,6 +43,8 @@ func TestPruningRemovesEverySurplusByItsRecordAndNothingThatDoesNotFit(t *testin {name: "a name that does not fit is left", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: []LedgerEntry{ {"tank", "before-move", "hourly", at(1)}, entry("tank", 2)}}, left: 1}, {name: "none removes nothing", keeps: `{"none":true}`, l: &ledgerCalls{entries: four}}, + {name: "a keep count of 0 is none: nothing removed, a warning names them (A-35)", keeps: `{"hourly":0,"daily":1}`, + l: &ledgerCalls{entries: four}, warns: "keep count of hourly is 0"}, {name: "paused removes nothing", keeps: `{"hourly":1}`, l: &ledgerCalls{entries: four}, setup: func(st memStates) { st.Put(StateSchedule, KeyPaused, map[string]string{}) }}, {name: "no read, no removal", keeps: `{"hourly":1}`, l: &ledgerCalls{read: errors.New("no bus")}}, @@ -81,6 +84,9 @@ func TestPruningRemovesEverySurplusByItsRecordAndNothingThatDoesNotFit(t *testin } f.on(line("zfs", SnapshotsArgv()), list.String()+"tank@by-hand\t1791400000\t1\t2\n") said, err := m.PruneScheduled(context.Background(), keepsOfJSON(t, c.keeps), false, "") + if c.warns != "" && !strings.Contains(strings.Join(m.Warnings, "|"), c.warns) { + t.Errorf("%s: warnings %v, want %q", c.name, m.Warnings, c.warns) + } if (err != nil) != c.wantErr { t.Errorf("%s: error %v", c.name, err) } From c0067e77fc596ad8dfa2716027e89cd9330ccf26 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:54:06 +0100 Subject: [PATCH 13/35] Z15 zfs-tools: zfs_config, zfs_snapshots_schedule, zfs_module_events, zfs_snapshots_check and zfs_findings read what the module owns, keeps and says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zfs_config also says what the module says before it leaves a node (Z17): the next load of the kernel module uses ZFS's defaults, the three units and /etc/hostid stay. Builds: design 42 zfs-owns-zfs §28, §30, §10, A-31, A-35 --- modules/zfs/cmd/zfs-tools/bus.go | 42 ++ modules/zfs/cmd/zfs-tools/machine.go | 6 +- modules/zfs/cmd/zfs-tools/main.go | 2 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 8 +- modules/zfs/cmd/zfs-tools/tools.go | 32 ++ modules/zfs/cmd/zfs-tools/tools_config.go | 447 ++++++++++++++++++ .../zfs/cmd/zfs-tools/tools_config_test.go | 237 ++++++++++ modules/zfs/module.json | 5 + 8 files changed, 775 insertions(+), 4 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/tools_config.go create mode 100644 modules/zfs/cmd/zfs-tools/tools_config_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index c846628ba..2760a8e64 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -163,3 +163,45 @@ func (busLedger) Entries() ([]LedgerEntry, error) { } return out, nil } + +// EventsStream is the module's events stream's local name in its `requires` (§25). +const EventsStream = "events" + +// busStream is the module's events stream through the tool runner. +type busStream struct{} + +// Last reads the stream from its first sequence on, page by page, and keeps the last n events of the kind. +func (busStream) Last(n int, kind string) ([]LoggedEvent, error) { + raw, err := stdio.BusState(buscalls.StreamFill, buscalls.StreamRequest{Stream: EventsStream}) + if err != nil { + return nil, err + } + var fill buscalls.StreamFilled + if err := json.Unmarshal(raw, &fill); err != nil { + return nil, err + } + var out []LoggedEvent + from := fill.First + for from != 0 && from <= fill.Last { + raw, err := stdio.BusState(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: EventsStream, From: from, Limit: 500}) + if err != nil { + return nil, err + } + var page buscalls.StreamPage + if err := json.Unmarshal(raw, &page); err != nil { + return nil, err + } + for _, msg := range page.Messages { + k := KindOfSubject(msg.Subject) + if kind != "" && k != kind { + continue + } + out = append(out, LoggedEvent{Sequence: msg.Sequence, Kind: k, At: msg.At, Body: msg.Body}) + if len(out) > n { + out = out[1:] + } + } + from = page.Next + } + return out, nil +} diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index ee817c47d..55ad575b1 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -139,6 +139,10 @@ type Machine struct { Emit func(eventType string, body any) error // States is the module's states on the bus (§26); nil where this process reaches none. States StateStore + // Exists says whether a path is there (a link counts); nil says no path is. + Exists func(path string) bool + // Log is the module's own events stream as the tools bundle reaches it; nil in a process. + Log EventLog // Errors is what the health check last saw of each device's error counts; nil judges every count // as past. Errors *ErrorMemory @@ -146,7 +150,7 @@ type Machine struct { // Here is the machine this process runs on. func Here() *Machine { - return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Node: os.Getenv("MESH_NODE"), WriteFile: writeLive, Lock: FileLock, Errors: LoadErrorMemory(ErrorsFile)} + return &Machine{Run: ExecRunner, UID: os.Getuid(), Now: time.Now, ReadFile: os.ReadFile, Resolve: filepath.EvalSymlinks, Node: os.Getenv("MESH_NODE"), WriteFile: writeLive, Lock: FileLock, Exists: func(p string) bool { _, err := os.Lstat(p); return err == nil }, Errors: LoadErrorMemory(ErrorsFile)} } // NotApplicable is the answer of a machine without ZFS: the module does not apply here. diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 0fb7282db..eff072e8f 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -67,7 +67,7 @@ func main() { } m := Here() m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) } - m.States, m.Call, m.Ledger, m.Await = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await + m.States, m.Call, m.Ledger, m.Await, m.Log = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await, busStream{} // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. if err := stdio.Serve("", Tools(m)); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index c7de1e430..bf01bec57 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -37,7 +37,11 @@ func readManifest(t *testing.T) (manifest, string) { // which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121). // handTools are the tools that start or steer the scheduled run, which replace no command. var handTools = map[string]bool{"zfs_snapshots_run_now": true, "zfs_snapshots_prune_now": true, "zfs_config_apply": true, - "zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true} + "zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true, + // The module's own read tools (§28) replace no command. + "zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true} + +var ownReads = map[string]bool{"zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true} func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) { m, raw := readManifest(t) @@ -45,7 +49,7 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T for _, tl := range Tools(newFake(t).machine()) { served = append(served, tl.Name) want := "(r)" - if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || handTools[tl.Name] { + if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || (handTools[tl.Name] && !ownReads[tl.Name]) { want = "(a)" } if !strings.HasSuffix(tl.Description, want) { diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go index 9d4ff7f44..8fee83e58 100644 --- a/modules/zfs/cmd/zfs-tools/tools.go +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -106,6 +106,33 @@ func Tools(m *Machine) []stdio.Tool { return map[string]any{"recorded": true}, nil }, }, + readTool("zfs_config", "Every value the module owns with its config value, what runs on the node, and how it stands: not managed, as set, "+ + "not applied yet, waiting (on what, since when), refused (why), failed, or taking effect at the next load of the kernel module; the "+ + "package's files the module never writes (with a .pacnew beside them) and what the module says before it leaves a node: the next load of "+ + "the kernel module uses ZFS's defaults, the three units stay, /etc/hostid stays. Changes nothing. (r)", nil, + func(map[string]any) (any, error) { return m.ConfigNow(ctx) }), + readTool("zfs_snapshots_schedule", "The schedule of the scheduled ZFS snapshots: the keep counts, when each kind next falls due, whether it is paused, "+ + "the entries past their keep count the next run would remove, and the ledger `{\"ledger\": [...]}` of every scheduled ZFS snapshot the run took "+ + "and did not remove. The scheduled run reads the ledger from here. Changes nothing. (r)", nil, + func(map[string]any) (any, error) { return m.SnapshotSchedule() }), + readTool("zfs_module_events", "The module's own events, newest first, from its events stream (config.applied, config.refused, config.waiting, "+ + "snapshot.taken.person and .schedule, snapshot.removed.person and .schedule, snapshot.skipped.person and .schedule, run.skipped, scrub.requested), "+ + "the last N of one kind or of all, with how many of each kind. Not ZFS's own events: zfs_events reads those. Changes nothing. (r)", + map[string]any{"last": map[string]any{"type": "integer", "description": "how many to show (default 20, 1 to 500)"}, + "kind": map[string]any{"type": "string", "description": "one kind of event (optional: every kind when left out)"}}, + func(a map[string]any) (any, error) { + n, err := whole(a, "last", 20, 1, 500) + if err != nil { + return nil, err + } + return m.ModuleEvents(n, text(a, "kind")) + }), + readTool("zfs_snapshots_check", "Compares the ledger with what ZFS holds: names each ledger entry whose ZFS snapshot is gone and each ZFS snapshot "+ + "named mesh- that the ledger does not hold. Changes nothing. (r)", nil, + func(map[string]any) (any, error) { return m.CheckSnapshots(ctx) }), + readTool("zfs_findings", "The event follower's findings: one key for each pool or device that is not ONLINE, deleted when it is ONLINE again or "+ + "leaves the pool. Changes nothing. (r)", nil, + func(map[string]any) (any, error) { return m.FindingsNow() }), { Name: "zfs_pools", Description: "Every imported pool's health in a line: its state (ONLINE, DEGRADED, FAULTED, …) with status's own " + @@ -311,3 +338,8 @@ func handTool(m *Machine, name, act, description string) stdio.Tool { }, } } + +// readTool is one of the module's own read tools. +func readTool(name, description string, input map[string]any, run func(map[string]any) (any, error)) stdio.Tool { + return stdio.Tool{Name: name, Description: description, Input: input, Run: run} +} diff --git a/modules/zfs/cmd/zfs-tools/tools_config.go b/modules/zfs/cmd/zfs-tools/tools_config.go new file mode 100644 index 000000000..156b4ce39 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/tools_config.go @@ -0,0 +1,447 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// The module's own read tools (design 42 zfs-owns-zfs §28, §30): what is owned and how it stands, the +// schedule of the scheduled ZFS snapshots with the ledger, the module's events, the ledger against ZFS, and the +// follower's findings. None of them changes anything. + +// configKeys are the config values the module owns, in the order they are said. +var configKeys = []string{"pool", "scrub-cadence", "scrub-day", "scrub-time", "arc-max", "arc-min", "module-options", "hostid", + "zed", "zed-handlers", "trim-schedule", "trim-day", "trim-time", "cache-devices", "log-devices", "log-mirror", + "pool-properties", "dataset-properties", "snapshots"} + +// appliedKeys are the values the apply converges (the others are the node-engine's files or the run's schedule): +// one of them set and with no key in the state `applied` has not been applied yet. +var appliedKeys = map[string]bool{"arc-max": true, "arc-min": true, "module-options": true, "hostid": true, "cache-devices": true, + "log-devices": true, "log-mirror": true, "pool-properties": true, "dataset-properties": true} + +// The states a value is shown in (§28). +const ( + StateNotManaged = "not managed" + StateAsSet = "as set" + StateNotYet = "not applied yet" + StateWaiting = "waiting" + StateRefused = "refused" + StateFailed = "failed" + StateAwaitingLoad = "takes effect at the next load of the kernel module" +) + +// ConfigValue is one owned config value as zfs_config shows it. +type ConfigValue struct { + Key string `json:"key"` + // Value is what the config file holds, absent when the value is not managed. + Value json.RawMessage `json:"value,omitempty"` + // Running is what runs on the node, where it is read live; else what the last apply read back. + Running string `json:"running,omitempty"` + State string `json:"state"` + Why string `json:"why,omitempty"` + WaitsOn string `json:"waits-on,omitempty"` + Since string `json:"since,omitempty"` + Note string `json:"note,omitempty"` +} + +// ConfigFileState is one of the package's files the module never writes (§10). +type ConfigFileState struct { + Path string `json:"path"` + Present bool `json:"present"` + // Pacnew says a `.pacnew` lies beside it: the package shipped a newer file the module did not take. + Pacnew bool `json:"pacnew,omitempty"` +} + +// ConfigAnswer is what zfs_config answers. +type ConfigAnswer struct { + Values []ConfigValue `json:"values"` + Files []ConfigFileState `json:"files"` + Leaving []string `json:"leaving"` + Said []string `json:"said"` +} + +// packageFiles are the files of §10: the package's, which the module never writes, and ZFS's own caches. +var packageFiles = []string{"/etc/default/zfs", "/etc/zfs/zed.d", "/etc/zfs/zed.d/zed.rc", "/etc/zfs/zpool.cache", "/etc/zfs/zfs-list.cache"} + +// leavingSaid is what the module says before it leaves a node (§30). +var leavingSaid = []string{ + "When the module leaves, the next load of the kernel module uses ZFS's own defaults: zfs.conf is removed.", + "The units zfs-import-cache.service, zfs-mount.service and zfs.target stay running and enabled; they are never stopped or disabled.", + "/etc/hostid stays: it is no resource the node-engine removes, and removing it would change the node's hostid, so that the pool's next import could need zpool import -f.", + "Pool and dataset properties and devices stay as the last apply set them.", + "The distribution's scrub and trim timers are put back as the module found them.", +} + +func (m *Machine) exists(path string) bool { return m.Exists != nil && m.Exists(path) } + +func rawManaged(raw json.RawMessage) bool { + t := strings.TrimSpace(string(raw)) + switch t { + case "", "null", `""`, "[]", "{}": + return false + } + return true +} + +// live is a kernel parameter read as it runs. +func (m *Machine) live(name string) (string, bool) { + b, err := m.ReadFile(ParametersDir + name) + if err != nil { + return "", false + } + return strings.TrimSpace(string(b)), true +} + +// worse orders the outcomes of a value's keys: the one a person must see first. +var worse = map[string]int{OutcomeApplied: 1, OutcomeAwaitingLoad: 2, OutcomeWaiting: 3, OutcomeFailed: 4, OutcomeRefused: 5} + +// ConfigNow shows every value the module owns: what the config file holds, what runs, how it stands, and the +// files it never writes. +func (m *Machine) ConfigNow(ctx context.Context) (ConfigAnswer, error) { + a := ConfigAnswer{Values: []ConfigValue{}, Files: []ConfigFileState{}, Leaving: leavingSaid, Said: []string{}} + text, err := m.ReadFile(ConfigFile) + if err != nil { + return a, fmt.Errorf("%s could not be read: %v", ConfigFile, err) + } + raw := map[string]json.RawMessage{} + if err := json.Unmarshal(text, &raw); err != nil { + return a, fmt.Errorf("%s is not a JSON object: %v", ConfigFile, err) + } + applied := map[string]AppliedEntry{} + if m.States != nil { + keys, err := m.States.Keys(StateApplied) + if err != nil { + return a, fmt.Errorf("the state %s could not be read: %v", StateApplied, err) + } + for _, k := range keys { + if v, ok, _ := m.States.Get(StateApplied, k); ok { + var e AppliedEntry + if json.Unmarshal(v, &e) == nil { + applied[k] = e + } + } + } + } + for _, key := range configKeys { + v := ConfigValue{Key: key, State: StateNotManaged} + if !rawManaged(raw[key]) { + a.Values = append(a.Values, v) + continue + } + v.Value = raw[key] + v.State = StateAsSet + if appliedKeys[key] { + v.State = StateNotYet + } + best := 0 + var afters []string + for _, k := range sortedKeys(applied) { + if first, _, _ := strings.Cut(k, "."); first != key { + continue + } + e := applied[k] + if e.After != "" { + afters = append(afters, k+"="+e.After) + } + if worse[e.Outcome] > best { + best = worse[e.Outcome] + v.Why, v.WaitsOn = e.Why, e.WaitsOn + v.Since = "" + if !e.Since.IsZero() { + v.Since = e.Since.Format(time.RFC3339) + } + switch e.Outcome { + case OutcomeApplied: + v.State = StateAsSet + case OutcomeAwaitingLoad: + v.State = StateAwaitingLoad + case OutcomeWaiting: + v.State = StateWaiting + case OutcomeFailed: + v.State = StateFailed + case OutcomeRefused: + v.State = StateRefused + } + } + } + v.Running = strings.Join(afters, ", ") + switch key { + case "arc-max", "arc-min": + if n, ok := m.live("zfs_" + strings.ReplaceAll(key, "-", "_")); ok { + v.Running = n + " bytes" + } + case "hostid": + if b, err := m.ReadFile(HostIDFile); err == nil { + if id, ok := nativeHostID(b); ok { + v.Running = fmt.Sprintf("%08x", id) + } + } + v.Note = "a change of the node's ZFS id needs a check of the pool's import at the next boot; the module never reboots" + } + a.Values = append(a.Values, v) + } + for _, p := range packageFiles { + f := ConfigFileState{Path: p, Present: m.exists(p), Pacnew: m.exists(p + ".pacnew")} + a.Files = append(a.Files, f) + if f.Pacnew { + a.Said = append(a.Said, p+".pacnew lies beside it: the package shipped a newer file the module does not take.") + } + } + a.Said = append([]string{fmt.Sprintf("%d of %d values are set.", countSet(a.Values), len(a.Values))}, a.Said...) + return a, nil +} + +func countSet(vs []ConfigValue) int { + n := 0 + for _, v := range vs { + if v.State != StateNotManaged { + n++ + } + } + return n +} + +// ScheduleAnswer is what zfs_snapshots_schedule answers. +type ScheduleAnswer struct { + Pool string `json:"pool,omitempty"` + Keeps map[string]int `json:"keeps"` + Paused map[string]any `json:"paused,omitempty"` + // Next is the moment each kind with a keep count next falls due; the run fires every quarter of an hour, so the + // ZFS snapshot is taken at the first run at or after it. + Next map[string]string `json:"next"` + // WouldRemove are the ledger's entries past their keep count: what the next run removes, if it passes its checks. + WouldRemove []LedgerEntry `json:"would-remove"` + Ledger []LedgerEntry `json:"ledger"` + Said []string `json:"said"` +} + +// nextDue is the first moment after now at which a kind falls due, in now's time zone. +func nextDue(kind string, now time.Time) time.Time { + day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + switch kind { + case "hourly": + return now.Truncate(time.Hour).Add(time.Hour) + case "weekly": + d := day.AddDate(0, 0, 1) + for d.Weekday() != time.Monday { + d = d.AddDate(0, 0, 1) + } + return d + case "monthly": + return time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, now.Location()) + } + return day.AddDate(0, 0, 1) +} + +// wouldRemove are the entries past their dataset's and kind's keep count; a kind with 0 removes nothing (A-35). +func wouldRemove(entries []LedgerEntry, k *Keeps) []LedgerEntry { + groups := map[string][]LedgerEntry{} + for _, e := range entries { + g := e.Dataset + " " + e.Kind + groups[g] = append(groups[g], e) + } + out := []LedgerEntry{} + for _, g := range sortedKeys(groups) { + list := groups[g] + if n := k.Of(list[0].Kind); n > 0 { + out = append(out, surplus(list, n)...) + } + } + return out +} + +// SnapshotSchedule shows the schedule of the scheduled ZFS snapshots, its next run, what the next run would remove and +// the ledger. The ledger is shown even when the config says nothing of it: the scheduled run reads it from here. +func (m *Machine) SnapshotSchedule() (ScheduleAnswer, error) { + a := ScheduleAnswer{Keeps: map[string]int{}, Next: map[string]string{}, WouldRemove: []LedgerEntry{}, Ledger: []LedgerEntry{}, Said: []string{}} + if m.Ledger == nil { + return a, fmt.Errorf("this process reaches no ledger") + } + entries, err := m.Ledger.Entries() + if err != nil { + return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) + } + if entries != nil { + a.Ledger = entries + } + if m.States != nil { + if raw, ok, err := m.States.Get(StateSchedule, KeyPaused); err == nil && ok { + _ = json.Unmarshal(raw, &a.Paused) + a.Said = append(a.Said, "The schedule is paused: nothing is taken or removed until zfs_snapshots_resume.") + } + } + text, err := m.ReadFile(ConfigFile) + if err != nil { + a.Said = append(a.Said, ConfigFile+" could not be read: "+err.Error()) + return a, nil + } + c, refused := ParseConfig(text, m.memMiB()) + switch { + case len(refused) > 0: + a.Said = append(a.Said, "The config is refused ("+refused[0].Key+" "+refused[0].Reason+"): the schedule is not known.") + case c.Snapshots == nil: + a.Said = append(a.Said, "snapshots is not managed: nothing is taken or removed.") + case c.Snapshots.None: + a.Pool = c.Pool + a.Said = append(a.Said, "snapshots is none: nothing is taken, and the scheduled ZFS snapshots that exist are kept.") + default: + a.Pool = c.Pool + for _, kind := range keepKinds { + a.Keeps[kind] = c.Snapshots.Of(kind) + if a.Keeps[kind] > 0 { + a.Next[kind] = nextDue(kind, m.Now()).Format(time.RFC3339) + } + } + a.WouldRemove = wouldRemove(a.Ledger, c.Snapshots) + a.Said = append(a.Said, fmt.Sprintf("%d scheduled ZFS snapshots are in the ledger; the next run would remove %d.", len(a.Ledger), len(a.WouldRemove))) + } + return a, nil +} + +// LoggedEvent is one event of the module's stream. +type LoggedEvent struct { + Sequence uint64 `json:"sequence"` + Kind string `json:"kind"` + At time.Time `json:"at"` + Body json.RawMessage `json:"body,omitempty"` +} + +// EventLog is the module's own events stream as the tools bundle reaches it (§25). +type EventLog interface { + // Last are the last n events, oldest first, only those of the kind when one is named. + Last(n int, kind string) ([]LoggedEvent, error) +} + +// EventKinds are the module's kinds of event (§25). +var EventKinds = []string{EventApplied, EventRefused, EventWaiting, EventTakenPerson, EventTakenSchedule, EventRemovedPerson, + EventRemovedSched, EventSkippedPerson, EventSkippedSched, EventRunSkipped, EventScrubRequested} + +// KindOfSubject is the kind an event's subject ends in. +func KindOfSubject(subject string) string { + for _, k := range EventKinds { + if subject == k || strings.HasSuffix(subject, "."+k) { + return k + } + } + return subject +} + +// ModuleEventsAnswer is what zfs_module_events answers. +type ModuleEventsAnswer struct { + Events []LoggedEvent `json:"events"` + ByKind map[string]int `json:"by-kind"` + Said []string `json:"said"` +} + +// ModuleEvents are the last n events of the module's stream, newest first, of one kind when it is named. +func (m *Machine) ModuleEvents(n int, kind string) (ModuleEventsAnswer, error) { + a := ModuleEventsAnswer{Events: []LoggedEvent{}, ByKind: map[string]int{}, Said: []string{}} + if kind != "" { + known := false + for _, k := range EventKinds { + known = known || k == kind + } + if !known { + return a, fmt.Errorf("%q is not a kind of this module's events: %s", kind, strings.Join(EventKinds, ", ")) + } + } + if m.Log == nil { + return a, fmt.Errorf("this process reaches no events stream") + } + evs, err := m.Log.Last(n, kind) + if err != nil { + return a, fmt.Errorf("the events stream could not be read: %v", err) + } + for i := len(evs) - 1; i >= 0; i-- { + a.Events = append(a.Events, evs[i]) + a.ByKind[evs[i].Kind]++ + } + a.Said = append(a.Said, fmt.Sprintf("%d events shown, newest first; the stream keeps 30 days.", len(a.Events))) + return a, nil +} + +// CheckAnswer is what zfs_snapshots_check answers. +type CheckAnswer struct { + // Gone are the ledger's entries whose ZFS snapshot ZFS no longer has. + Gone []LedgerEntry `json:"gone"` + // Unrecorded are the snapshots named mesh- that ZFS has and the ledger does not. + Unrecorded []string `json:"unrecorded"` + Said []string `json:"said"` +} + +// CheckSnapshots compares the ledger with what ZFS holds; it changes nothing. +func (m *Machine) CheckSnapshots(ctx context.Context) (CheckAnswer, error) { + a := CheckAnswer{Gone: []LedgerEntry{}, Unrecorded: []string{}, Said: []string{}} + if m.Ledger == nil { + return a, fmt.Errorf("this process reaches no ledger") + } + entries, err := m.Ledger.Entries() + if err != nil { + return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) + } + snaps, err := m.Snapshots(ctx) + if err != nil { + return a, err + } + have := map[string]bool{} + for _, s := range snaps { + have[s.Name] = true + } + recorded := map[string]bool{} + for _, e := range entries { + full := e.Dataset + "@" + e.Name + recorded[full] = true + if !have[full] { + a.Gone = append(a.Gone, e) + } + } + for _, s := range snaps { + if strings.HasPrefix(s.Snap, MeshPrefix) && !recorded[s.Name] { + a.Unrecorded = append(a.Unrecorded, s.Name) + } + } + sort.Strings(a.Unrecorded) + a.Said = append(a.Said, fmt.Sprintf("%d ledger entries, %d with no ZFS snapshot, %d mesh- snapshots not in the ledger. Nothing was changed.", + len(entries), len(a.Gone), len(a.Unrecorded))) + return a, nil +} + +// FindingsAnswer is what zfs_findings answers. +type FindingsAnswer struct { + Findings []FindingKey `json:"findings"` + Said []string `json:"said"` +} + +// FindingKey is one key of the follower's state `findings`. +type FindingKey struct { + Key string `json:"key"` + Value json.RawMessage `json:"value"` +} + +// FindingsNow are the follower's findings: one key per pool or device not ONLINE. +func (m *Machine) FindingsNow() (FindingsAnswer, error) { + a := FindingsAnswer{Findings: []FindingKey{}, Said: []string{}} + if m.States == nil { + return a, fmt.Errorf("this process reaches no state") + } + keys, err := m.States.Keys(StateFindings) + if err != nil { + return a, fmt.Errorf("the state %s could not be read: %v", StateFindings, err) + } + sort.Strings(keys) + for _, k := range keys { + if v, ok, err := m.States.Get(StateFindings, k); err == nil && ok { + a.Findings = append(a.Findings, FindingKey{Key: k, Value: v}) + } + } + if len(a.Findings) == 0 { + a.Said = append(a.Said, "No pool or device is not ONLINE: the follower holds no finding.") + } else { + a.Said = append(a.Said, fmt.Sprintf("%d findings: a pool or device not ONLINE.", len(a.Findings))) + } + return a, nil +} diff --git a/modules/zfs/cmd/zfs-tools/tools_config_test.go b/modules/zfs/cmd/zfs-tools/tools_config_test.go new file mode 100644 index 000000000..5e55fca27 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/tools_config_test.go @@ -0,0 +1,237 @@ +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +type staticLedger struct { + entries []LedgerEntry + err error +} + +func (l staticLedger) Apply(string, string, any, string) error { return nil } +func (l staticLedger) Entries() ([]LedgerEntry, error) { return l.entries, l.err } + +type staticLog struct { + evs []LoggedEvent + err error +} + +func (l staticLog) Last(n int, kind string) ([]LoggedEvent, error) { + var out []LoggedEvent + for _, e := range l.evs { + if kind == "" || e.Kind == kind { + out = append(out, e) + } + } + if len(out) > n { + out = out[len(out)-n:] + } + return out, l.err +} + +// §28, §30, §10, §29: zfs_config shows each owned value with its config value, what runs, and how it stands +// (not managed, as set, not applied yet, waiting, refused, awaiting a load), the package's files with a .pacnew +// beside them, and what the module says before it leaves a node. +func TestZfsConfigShowsEveryValueHowItStandsAndWhatTheModuleSaysBeforeLeaving(t *testing.T) { + f := newFake(t) + m := f.machine() + st := memStates{} + m.States = st + f.files[ConfigFile] = `{"pool":"tank","arc-max":4096,"module-options":["zfs_txg_timeout=5"],"hostid":"0a0b0c0d","cache-devices":["x"], + "log-devices":[],"zed":"on","trim-schedule":"","snapshots":{"daily":3}}` + f.files[ParametersDir+"zfs_arc_max"] = "4294967296\n" + hb := make([]byte, 4) + binary.NativeEndian.PutUint32(hb, 0x0a0b0c0d) + f.files[HostIDFile] = string(hb) + m.Exists = func(p string) bool { return p == "/etc/default/zfs" || p == "/etc/default/zfs.pacnew" } + st.Put(StateApplied, "arc-max", AppliedEntry{Before: "1", After: "4294967296", Outcome: OutcomeApplied}) + st.Put(StateApplied, "module-options.zfs_txg_timeout", AppliedEntry{After: "5", Outcome: OutcomeAwaitingLoad}) + st.Put(StateApplied, "hostid", AppliedEntry{After: "0a0b0c0d", Outcome: OutcomeRefused, Why: "the pool's import would need -f"}) + st.Put(StateApplied, "cache-devices", AppliedEntry{Outcome: OutcomeWaiting, WaitsOn: "a scrub of tank", Since: f.now}) + st.Put(StateApplied, "trim-day", AppliedEntry{Outcome: OutcomeApplied}) // a value gone back to not managed is not shown + a, err := m.ConfigNow(context.Background()) + if err != nil { + t.Fatal(err) + } + by := map[string]ConfigValue{} + for _, v := range a.Values { + by[v.Key] = v + } + want := map[string]string{"pool": StateAsSet, "arc-max": StateAsSet, "module-options": StateAwaitingLoad, "hostid": StateRefused, + "cache-devices": StateWaiting, "log-devices": StateNotManaged, "zed": StateAsSet, "trim-schedule": StateNotManaged, + "trim-day": StateNotManaged, "snapshots": StateAsSet, "arc-min": StateNotManaged} + for k, w := range want { + if by[k].State != w { + t.Errorf("%s: %q, want %q", k, by[k].State, w) + } + } + if len(a.Values) != len(configKeys) { + t.Errorf("%d values for %d keys", len(a.Values), len(configKeys)) + } + if by["arc-max"].Running != "4294967296 bytes" || by["hostid"].Running != "0a0b0c0d" || by["hostid"].Why == "" || + !strings.Contains(by["hostid"].Note, "check of the pool's import") || by["cache-devices"].WaitsOn != "a scrub of tank" || by["cache-devices"].Since == "" { + t.Errorf("values: %+v", by) + } + if by["arc-min"].Value != nil { + t.Errorf("an unmanaged value shows %s", by["arc-min"].Value) + } + pacnew := false + for _, fl := range a.Files { + pacnew = pacnew || (fl.Path == "/etc/default/zfs" && fl.Present && fl.Pacnew) + } + all := strings.Join(a.Leaving, " ") + if !pacnew || !strings.Contains(strings.Join(a.Said, " "), "/etc/default/zfs.pacnew") || len(a.Files) != len(packageFiles) { + t.Errorf("files %+v said %v", a.Files, a.Said) + } + for _, s := range []string{"ZFS's own defaults", "zfs-import-cache.service, zfs-mount.service and zfs.target stay", "/etc/hostid stays"} { + if !strings.Contains(all, s) { + t.Errorf("leaving does not say %q: %v", s, a.Leaving) + } + } + delete(f.files, ConfigFile) + if _, err := m.ConfigNow(context.Background()); err == nil { + t.Error("an unreadable config file was answered") + } +} + +// §28, A-31: zfs_snapshots_schedule answers the keep counts, each kind's next moment, the pause, what the next +// run would remove (a keep count of 0 removes nothing, A-35) and the ledger as `{"ledger": [...]}`, which the run reads. +func TestSnapshotsScheduleAnswersTheLedgerTheRunReads(t *testing.T) { + e := func(h int) LedgerEntry { + return LedgerEntry{"tank", "mesh-hourly-x" + string(rune('0'+h)), "hourly", time.Date(2026, 10, 9, h, 0, 0, 0, time.UTC).Format(time.RFC3339)} + } + old := LedgerEntry{"tank", "mesh-daily-old", "daily", "2026-09-01T00:00:00Z"} + cases := []struct { + name, conf string + l staticLedger + paused bool + remove int + next map[string]string + said string + wantErr bool + }{ + {name: "counts and surplus", conf: `{"pool":"tank","snapshots":{"hourly":2,"daily":0}}`, l: staticLedger{entries: []LedgerEntry{e(1), e(2), e(3), old}}, + remove: 1, next: map[string]string{"hourly": "2026-10-09T13:00:00"}, said: "would remove 1"}, + {name: "weekly and monthly", conf: `{"pool":"tank","snapshots":{"weekly":1,"monthly":1,"daily":1}}`, next: map[string]string{ + "daily": "2026-10-10T00:00:00", "weekly": "2026-10-12T00:00:00", "monthly": "2026-11-01T00:00:00"}}, + {name: "paused", conf: `{"pool":"tank","snapshots":{"hourly":1}}`, paused: true, said: "paused"}, + {name: "none keeps what exists", conf: `{"pool":"tank","snapshots":{"none":true}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "are kept"}, + {name: "not managed", conf: `{"pool":"tank"}`, said: "not managed"}, + {name: "refused config still answers the ledger", conf: `{"snapshots":{"hourly":999}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "refused"}, + {name: "ledger unreadable", conf: `{}`, l: staticLedger{err: errors.New("no bus")}, wantErr: true}, + } + for _, c := range cases { + f := newFake(t) + f.now = time.Date(2026, 10, 9, 12, 30, 0, 0, time.UTC) // a Friday + m := f.machine() + st := memStates{} + m.States, m.Ledger = st, c.l + f.files[ConfigFile] = c.conf + if c.paused { + st.Put(StateSchedule, KeyPaused, map[string]string{"by": "x", "why": "y"}) + } + a, err := m.SnapshotSchedule() + if (err != nil) != c.wantErr { + t.Errorf("%s: %v", c.name, err) + continue + } + if c.wantErr { + continue + } + if len(a.WouldRemove) != c.remove || (c.paused && a.Paused == nil) || !strings.Contains(strings.Join(a.Said, " "), c.said) { + t.Errorf("%s: %+v", c.name, a) + } + for k, v := range c.next { + if !strings.HasPrefix(a.Next[k], v) { + t.Errorf("%s: next %s %q, want %q", c.name, k, a.Next[k], v) + } + } + // The run reads the answer as it reads the ledger. + raw, _ := json.Marshal(a) + r := f.machine() + r.Call = func(key string, _ any) (json.RawMessage, error) { return raw, nil } + got, err := r.ledgerEntries() + if err != nil || len(got) != len(c.l.entries) { + t.Errorf("%s: the run reads %v, %v", c.name, got, err) + } + } +} + +// §28: zfs_module_events answers the last N events of the stream, newest first, of one kind when named, with how +// many of each; a kind the module does not say is refused. +func TestModuleEventsAreTheStreamsLastNNewestFirstByKind(t *testing.T) { + f := newFake(t) + m := f.machine() + at := func(i int) time.Time { return time.Date(2026, 10, 9, 10, i, 0, 0, time.UTC) } + m.Log = staticLog{evs: []LoggedEvent{{1, EventApplied, at(1), nil}, {2, EventRefused, at(2), nil}, {3, EventApplied, at(3), nil}, {4, EventRunSkipped, at(4), nil}}} + a, err := m.ModuleEvents(3, "") + if err != nil || len(a.Events) != 3 || a.Events[0].Sequence != 4 || a.ByKind[EventApplied] != 1 || a.ByKind[EventRunSkipped] != 1 { + t.Errorf("last 3: %+v %v", a, err) + } + if a, err := m.ModuleEvents(5, EventApplied); err != nil || len(a.Events) != 2 || a.Events[0].Sequence != 3 { + t.Errorf("one kind: %+v %v", a, err) + } + if _, err := m.ModuleEvents(5, "config.nothing"); err == nil { + t.Error("an unknown kind was answered") + } + m.Log = staticLog{err: errors.New("no bus")} + if _, err := m.ModuleEvents(5, ""); err == nil { + t.Error("a stream that cannot be read was answered") + } + for subject, kind := range map[string]string{"zfs.snapshot.taken.person": EventTakenPerson, "snapshot.taken.schedule": EventTakenSchedule, "other": "other"} { + if KindOfSubject(subject) != kind { + t.Errorf("%s is %s", subject, KindOfSubject(subject)) + } + } +} + +// §28: zfs_snapshots_check names each ledger entry whose ZFS snapshot is gone and each mesh- snapshot the ledger +// does not hold, and changes nothing. +func TestSnapshotsCheckNamesWhatTheLedgerAndZFSDoNotShare(t *testing.T) { + f := newFake(t) + f.on(line("zfs", SnapshotsArgv()), "tank@mesh-hourly-20261009T1000Z\t1791400000\t1\t2\ntank@mesh-hourly-20261009T1100Z\t1791400000\t1\t2\ntank@by-hand\t1791400000\t1\t2\n") + f.uid = 0 + m := f.machine() + m.Ledger = staticLedger{entries: []LedgerEntry{{"tank", "mesh-hourly-20261009T1000Z", "hourly", "2026-10-09T10:00:00Z"}, + {"tank", "mesh-hourly-20261009T0900Z", "hourly", "2026-10-09T09:00:00Z"}}} + a, err := m.CheckSnapshots(context.Background()) + if err != nil || len(a.Gone) != 1 || a.Gone[0].Name != "mesh-hourly-20261009T0900Z" || + len(a.Unrecorded) != 1 || a.Unrecorded[0] != "tank@mesh-hourly-20261009T1100Z" { + t.Errorf("%+v %v", a, err) + } + for _, c := range f.calls { + if strings.Contains(c, "destroy") || strings.Contains(c, " set ") || strings.Contains(c, "snapshot --") { + t.Errorf("a check changed something: %s", c) + } + } + m.Ledger = staticLedger{err: errors.New("no bus")} + if _, err := m.CheckSnapshots(context.Background()); err == nil { + t.Error("a ledger that cannot be read was answered") + } +} + +// §27, §28: zfs_findings shows the follower's keys, none when every device is ONLINE. +func TestFindingsAreTheFollowersKeys(t *testing.T) { + f := newFake(t) + m := f.machine() + st := memStates{} + m.States = st + if a, err := m.FindingsNow(); err != nil || len(a.Findings) != 0 || !strings.Contains(a.Said[0], "No pool or device") { + t.Errorf("%+v %v", a, err) + } + st.Put(StateFindings, "pool.tank.sda", map[string]string{"summary": "sda is FAULTED"}) + if a, err := m.FindingsNow(); err != nil || len(a.Findings) != 1 || a.Findings[0].Key != "pool.tank.sda" { + t.Errorf("%+v %v", a, err) + } + m.States = nil + if _, err := m.FindingsNow(); err == nil { + t.Error("no state, no answer") + } +} diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 431e2fb24..200e26364 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -65,6 +65,11 @@ "zfs_snapshots_pause", "zfs_snapshots_resume", "zfs_snapshots_record", + "zfs_config", + "zfs_snapshots_schedule", + "zfs_module_events", + "zfs_snapshots_check", + "zfs_findings", "zfs_io", "zfs_arc", "zfs_events", From 97bdb7d65635c8d57b00a86e0d4622f5b06486da Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:55:17 +0100 Subject: [PATCH 14/35] Z16 zfs-health: a refused or failed command in the applied state is unhealthy naming its key, and the module's own warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §29, §15, A-35 --- modules/zfs/cmd/zfs-tools/health.go | 8 +- modules/zfs/cmd/zfs-tools/health_module.go | 152 +++++++++++++++++++++ modules/zfs/cmd/zfs-tools/health_test.go | 71 ++++++++++ 3 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 modules/zfs/cmd/zfs-tools/health_module.go diff --git a/modules/zfs/cmd/zfs-tools/health.go b/modules/zfs/cmd/zfs-tools/health.go index 076d0d96a..7bfad37f7 100644 --- a/modules/zfs/cmd/zfs-tools/health.go +++ b/modules/zfs/cmd/zfs-tools/health.go @@ -37,6 +37,8 @@ type Look struct { ScheduleErr error // Served is the last slot the scheduled run served, per pool. Served map[string]int64 + // Module is what the module's own config, states and ledger say (§29); nil judges the pools alone. + Module *ModuleLook } // errorFinding judges one line of status's error counts against the last look: a count that rose is a @@ -174,6 +176,10 @@ func (mem *ErrorMemory) Judge(l Look) Health { if l.Schedule != nil && statusOf(l.Status, l.Schedule.Pool) == nil { warn = append(warn, fmt.Sprintf("The scrub schedule names pool %s, which is not imported here.", l.Schedule.Pool)) } + if l.Module != nil { + f, w := l.moduleFindings() + found, warn = append(found, f...), append(warn, w...) + } mem.save() if len(found) > 0 { return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn} @@ -205,7 +211,7 @@ func (m *Machine) Health(ctx context.Context) Health { } sched, serr := m.ScheduleNow(ctx) return m.Errors.Judge(Look{Pools: pools, Status: status, Now: m.Now(), Schedule: sched, ScheduleErr: serr, - Served: m.readServed(ServedFile)}) + Served: m.readServed(ServedFile), Module: m.moduleLook()}) } func sentence(s string) string { diff --git a/modules/zfs/cmd/zfs-tools/health_module.go b/modules/zfs/cmd/zfs-tools/health_module.go new file mode 100644 index 000000000..e975e32e9 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/health_module.go @@ -0,0 +1,152 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// What the module's own work adds to its health (design 42 zfs-owns-zfs §29, §15). +// +// Unhealthy: a `refused` (the hostid's among them) or a failed command in the state `applied`, naming the key and +// why. Warnings: a wait, naming on what and since when; a parameter awaiting a load of the kernel module; a +// `.pacnew` beside a package file; the last scheduled ZFS snapshot of a kind older than its cadence; `autoreplace` on +// or a spare in the pool while `zed` is not `on`; the pool above 80 % with `snapshots` set; scheduled ZFS snapshots +// kept after `snapshots` went to `none` (or a kind's keep count to 0); `paused` set. + +// ModuleLook is what one health check read of the module's own config, states and ledger. +type ModuleLook struct { + Config *Config + Applied map[string]AppliedEntry + Paused bool + // Ledger is the scheduled ZFS snapshots; LedgerKnown says whether it could be read. + Ledger []LedgerEntry + LedgerKnown bool + // Pacnew are the package files a `.pacnew` lies beside. + Pacnew []string +} + +// snapshotLate is how old the last scheduled ZFS snapshot of a kind may be before it is a warning: its cadence and +// the margin for a run that waited for the lock or a full pool. +var snapshotLate = map[string]time.Duration{"hourly": 2 * time.Hour, "daily": 36 * time.Hour, "weekly": 9 * 24 * time.Hour, + "monthly": 38 * 24 * time.Hour} + +// moduleLook reads what the module itself keeps. It answers nil when the config file is not readable or not valid: +// the module is then judged on its pools alone, and the refusal is already in the state `applied`. +func (m *Machine) moduleLook() *ModuleLook { + text, err := m.ReadFile(ConfigFile) + if err != nil { + return nil + } + c, refused := ParseConfig(text, m.memMiB()) + if len(refused) > 0 { + c = nil + } + l := &ModuleLook{Config: c, Applied: map[string]AppliedEntry{}} + if m.States != nil { + if keys, err := m.States.Keys(StateApplied); err == nil { + for _, k := range keys { + if raw, ok, _ := m.States.Get(StateApplied, k); ok { + var e AppliedEntry + if json.Unmarshal(raw, &e) == nil { + l.Applied[k] = e + } + } + } + } + } + l.Paused = m.Paused() + if m.Ledger != nil { + if es, err := m.Ledger.Entries(); err == nil { + l.Ledger, l.LedgerKnown = es, true + } + } + for _, p := range []string{"/etc/default/zfs", "/etc/zfs/zed.d/zed.rc"} { + if m.exists(p + ".pacnew") { + l.Pacnew = append(l.Pacnew, p) + } + } + return l +} + +// moduleFindings judges the module's own work. +func (l Look) moduleFindings() (found, warn []string) { + k := l.Module + for _, key := range sortedKeys(k.Applied) { + e := k.Applied[key] + switch e.Outcome { + case OutcomeRefused: + found = append(found, sentence(fmt.Sprintf("%s is refused: %s", key, e.Why))) + case OutcomeFailed: + found = append(found, sentence(fmt.Sprintf("a command for %s failed: %s", key, e.Why))) + case OutcomeWaiting: + since := "a time not said" + if !e.Since.IsZero() { + since = e.Since.Local().Format("2006-01-02 15:04") + } + warn = append(warn, sentence(fmt.Sprintf("%s waits on %s since %s", key, e.WaitsOn, since))) + case OutcomeAwaitingLoad: + warn = append(warn, sentence(fmt.Sprintf("%s takes effect at the next load of the kernel module", key))) + } + } + for _, p := range k.Pacnew { + warn = append(warn, sentence(p+".pacnew lies beside it: the package shipped a newer file the module does not take")) + } + if k.Paused { + warn = append(warn, "The scheduled ZFS snapshots are paused: none is taken or removed.") + } + c := k.Config + if c == nil { + return found, warn + } + if c.Zed != "on" { + spare := false + for _, p := range l.Status { + if p.Name == c.Pool { + if s := sectionOf(&p, "spare"); s != nil && len(s.Children) > 0 { + spare = true + } + } + } + if c.PoolProperties["autoreplace"] == "on" || spare { + warn = append(warn, "Pool "+c.Pool+" has autoreplace on or a spare, and zed is not on: nothing replaces a failed device by itself.") + } + } + if c.Snapshots != nil && !c.Snapshots.None { + for _, p := range l.Pools { + if p.Name == c.Pool && p.Capacity > FullPercent { + warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: no scheduled ZFS snapshot is taken.", p.Name, p.Capacity, FullPercent)) + } + } + } + if !k.LedgerKnown { + return found, warn + } + if c.Snapshots != nil && !c.Snapshots.None { + for _, kind := range keepsOf(c.Snapshots) { + var last time.Time + for _, e := range k.Ledger { + if at, err := time.Parse(time.RFC3339, e.At); err == nil && e.Kind == kind && at.After(last) { + last = at + } + } + if !last.IsZero() && l.Now.Sub(last) > snapshotLate[kind] { + warn = append(warn, fmt.Sprintf("The last scheduled %s ZFS snapshot is from %s, older than its cadence allows.", kind, last.Local().Format("2006-01-02 15:04"))) + } + } + } + kept := map[string][]string{} + for _, e := range k.Ledger { + if c.Snapshots == nil || c.Snapshots.None || c.Snapshots.Of(e.Kind) == 0 { + kept[e.Kind] = append(kept[e.Kind], e.Dataset+"@"+e.Name) + } + } + for _, kind := range keepKinds { + if len(kept[kind]) > 0 { + warn = append(warn, fmt.Sprintf("%d scheduled %s ZFS snapshots are kept though snapshots does not ask for them (none or a keep count of 0 removes nothing): %s.", + len(kept[kind]), kind, strings.Join(kept[kind], ", "))) + } + } + return found, warn +} diff --git a/modules/zfs/cmd/zfs-tools/health_test.go b/modules/zfs/cmd/zfs-tools/health_test.go index 875d8ae70..dcf8002fa 100644 --- a/modules/zfs/cmd/zfs-tools/health_test.go +++ b/modules/zfs/cmd/zfs-tools/health_test.go @@ -5,6 +5,7 @@ import ( "context" "strings" "testing" + "time" ) func TestOnlinePoolsWithoutErrorsAreHealthyAndAnOldScrubIsOnlyAWarning(t *testing.T) { @@ -108,3 +109,73 @@ func TestTheFollowerArgvAndAResilverStartAndFinishAreJournaled(t *testing.T) { t.Errorf("the resilver's start: %q", out.String()) } } + +// §29, §15: a refused (the hostid's among them) or failed command in the state `applied` is unhealthy naming the key +// and why, and healthy again once it is gone; each warning of §29 is a warning and leaves the module healthy. +func TestTheModulesOwnWorkAddsToItsHealthAsSection29Says(t *testing.T) { + st, _ := ParseStatus(fixture(t, "zpool-status-scrubbing.txt")) + now := newFake(t).now + snap := func(json string) *Config { + c, ref := ParseConfig([]byte(json), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + return c + } + entry := func(kind string, h int) LedgerEntry { + return LedgerEntry{"tank", "mesh-" + kind + "-x", kind, now.Add(-time.Duration(h) * time.Hour).UTC().Format(time.RFC3339)} + } + cases := []struct { + name string + look ModuleLook + capacity int + unhealthy string + warning string + }{ + {name: "hostid refused", look: ModuleLook{Applied: map[string]AppliedEntry{"hostid": {Outcome: OutcomeRefused, Why: "the pool's import would need -f"}}}, + unhealthy: "Hostid is refused: the pool's import would need -f"}, + {name: "a failed command", look: ModuleLook{Applied: map[string]AppliedEntry{"cache-devices": {Outcome: OutcomeFailed, Why: "zpool add failed"}}}, + unhealthy: "a command for cache-devices failed: zpool add failed"}, + {name: "a wait", look: ModuleLook{Applied: map[string]AppliedEntry{"cache-devices": {Outcome: OutcomeWaiting, WaitsOn: "a scrub of tank", Since: now}}}, + warning: "waits on a scrub of tank since"}, + {name: "awaiting a load", look: ModuleLook{Applied: map[string]AppliedEntry{"module-options.zfs_txg_timeout": {Outcome: OutcomeAwaitingLoad}}}, + warning: "takes effect at the next load of the kernel module"}, + {name: "a pacnew", look: ModuleLook{Pacnew: []string{"/etc/default/zfs"}}, warning: "/etc/default/zfs.pacnew"}, + {name: "paused", look: ModuleLook{Paused: true}, warning: "are paused"}, + {name: "last snapshot older than its cadence", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"hourly":2}}`), LedgerKnown: true, + Ledger: []LedgerEntry{entry("hourly", 5)}}, warning: "last scheduled hourly ZFS snapshot"}, + {name: "autoreplace without zed", look: ModuleLook{Config: snap(`{"pool":"tank","pool-properties":{"autoreplace":"on"}}`)}, warning: "autoreplace on or a spare"}, + {name: "pool above 80 % with snapshots", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"daily":2}}`)}, capacity: 85, warning: "no scheduled ZFS snapshot is taken"}, + {name: "kept after none", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"none":true}}`), LedgerKnown: true, Ledger: []LedgerEntry{entry("daily", 1)}}, + warning: "kept though snapshots does not ask for them"}, + {name: "kept at a keep count of 0", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"hourly":0,"daily":1}}`), LedgerKnown: true, + Ledger: []LedgerEntry{entry("hourly", 1), entry("daily", 1)}}, warning: "1 scheduled hourly ZFS snapshots are kept"}, + {name: "nothing to say", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"daily":2}}`), LedgerKnown: true, Ledger: []LedgerEntry{entry("daily", 1)}}}, + } + for _, c := range cases { + look := c.look + if look.Applied == nil { + look.Applied = map[string]AppliedEntry{} + } + cap := c.capacity + if cap == 0 { + cap = 50 + } + h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Capacity: cap, Health: "ONLINE"}}, Status: st, Now: now, Module: &look}) + if c.unhealthy != "" { + if h.Healthy || !strings.Contains(strings.ToLower(h.Why), strings.ToLower(c.unhealthy)) { + t.Errorf("%s: %+v", c.name, h) + } + continue + } + all := strings.Join(h.Warnings, " ") + if !h.Healthy || (c.warning == "" && len(h.Warnings) != 0) || (c.warning != "" && !strings.Contains(all, c.warning)) { + t.Errorf("%s: %+v", c.name, h) + } + } + // Once the refusal is gone the module is healthy again. + if h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Capacity: 50, Health: "ONLINE"}}, Status: st, Now: now, + Module: &ModuleLook{Applied: map[string]AppliedEntry{}}}); !h.Healthy { + t.Errorf("after the refusal: %+v", h) + } +} From 5ab8bb7e62bce7418e1a4c2cef3f751c31faa4c7 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:00:42 +0100 Subject: [PATCH 15/35] Z1 Z2 Z3 zfs-manifest: packages and values, files and units, the apply and the run in the declared order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest declares zfs-utils and zfs-dkms, renames scrub-pool to pool, and declares every new value not managed until set; its config file is ${config-json}, the options file, ZED's folder, rc, functions, handlers and drop-in each under their condition, the three ZFS units, the apply (run-once, restarted by its config file) and the run before the distribution's scrub and trim timers. One commit: the three subtasks change the one file module.json. Builds: design 42 zfs-owns-zfs §6, §7, §8, §9, §11, §12, §13, §14, §15, §18, §20, A-1, A-2, A-3, A-7, A-14, A-15, A-16, A-23 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 314 ++++--- modules/zfs/cmd/zfs-tools/templates_test.go | 32 + modules/zfs/files/all-syslog.sh | 52 ++ modules/zfs/files/config.json | 1 + .../zfs/files/resilver_finish-start-scrub.sh | 20 + modules/zfs/files/scrub.conf | 6 - modules/zfs/files/zed-functions.sh | 825 ++++++++++++++++++ modules/zfs/files/zed-mesh.conf | 4 + modules/zfs/files/zed.rc | 7 + modules/zfs/files/zfs.conf | 5 + modules/zfs/module.json | 459 ++++++++-- 11 files changed, 1526 insertions(+), 199 deletions(-) create mode 100644 modules/zfs/files/all-syslog.sh create mode 100644 modules/zfs/files/config.json create mode 100644 modules/zfs/files/resilver_finish-start-scrub.sh delete mode 100644 modules/zfs/files/scrub.conf create mode 100644 modules/zfs/files/zed-functions.sh create mode 100644 modules/zfs/files/zed-mesh.conf create mode 100644 modules/zfs/files/zed.rc create mode 100644 modules/zfs/files/zfs.conf diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index bf01bec57..c8e5f7a45 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -2,13 +2,13 @@ package main import ( "encoding/json" + "fmt" "os" "path/filepath" "regexp" "sort" "strings" "testing" - "time" ) type manifest struct { @@ -84,15 +84,203 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T } } -// ADR 0240: zfs_health on the watch, which runs as the operator account; the follower judged by its -// unit. The scheduled run is not long-running and declares none. The module installs nothing, so it never -// puts ZFS on a machine. -func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { - m, raw := readManifest(t) - byID := map[string]map[string]any{} - for _, r := range m.Resources { - byID[r["id"].(string)] = r +// manifestOf is the whole manifest as a tree, for the tests that read a field the typed view does not. +func manifestOf(t *testing.T) (map[string]any, map[string]map[string]any, map[string]int) { + t.Helper() + _, raw := readManifest(t) + var tree map[string]any + if err := json.Unmarshal([]byte(raw), &tree); err != nil { + t.Fatal(err) } + byID, order := map[string]map[string]any{}, map[string]int{} + for i, r := range tree["resources"].([]any) { + res := r.(map[string]any) + byID[res["id"].(string)] = res + order[res["id"].(string)] = i + } + return tree, byID, order +} + +// conditions are the `if-config` of a resource, as written. +func conditions(r map[string]any) string { + var out []string + l, _ := r["if-config"].([]any) + for _, c := range l { + out = append(out, c.(string)) + } + return strings.Join(out, ",") +} + +// §7, §8 (A-1), §11 (A-14): the two packages are declared and the debug one is not; `pool` renames `scrub-pool`; each new value is +// not managed until set (no default, not required), of the form and trust §11 lists. +func TestTheManifestDeclaresThePackagesAndEveryValueAsSections7To11Say(t *testing.T) { + tree, byID, _ := manifestOf(t) + for id, pkg := range map[string]string{"utils": "zfs-utils", "dkms": "zfs-dkms"} { + if r := byID[id]; r == nil || r["type"] != "package" || r["package"] != pkg || r["state"] != nil { + t.Errorf("%s: %v", id, r) + } + } + for _, r := range byID { + if r["package"] == "zfs-utils-debug" { + t.Errorf("the debug package is not the module's: %v", r) + } + } + cfg := tree["config"].(map[string]any) + if _, old := cfg["scrub-pool"]; old { + t.Errorf("scrub-pool is declared; it is only in renamed-from") + } + if p := cfg["pool"].(map[string]any); p["renamed-from"] != "scrub-pool" || p["type"] != "string" || p["trusted"] != true || p["required"] != true || p["default"] != nil { + t.Errorf("pool: %v", p) + } + for k, want := range map[string]string{"scrub-cadence": "string/false", "scrub-day": "string/false", "scrub-time": "string/false"} { + if v := cfg[k].(map[string]any); v["default"] == nil || fmt.Sprint(v["type"]) != strings.Split(want, "/")[0] { + t.Errorf("%s keeps its default: %v", k, v) + } + } + new := map[string]string{"arc-max": "number/false", "arc-min": "number/false", "module-options": "list/true", "hostid": "string/true", "zed": "string/false", + "zed-handlers": "list/true", "trim-schedule": "string/false", "trim-day": "string/false", "trim-time": "string/false", "cache-devices": "list/true", + "log-devices": "list/true", "log-mirror": "boolean/true", "pool-properties": "object/true", "dataset-properties": "object/true", "snapshots": "object/true"} + for k, want := range new { + v, ok := cfg[k].(map[string]any) + if !ok { + t.Errorf("%s is not declared", k) + continue + } + if got := fmt.Sprint(v["type"], "/", v["trusted"]); got != want || v["default"] != nil || v["required"] != nil { + t.Errorf("%s: %s with default %v required %v, want %s not managed until set", k, got, v["default"], v["required"], want) + } + } + if len(cfg) != len(new)+4 { + t.Errorf("%d values declared, want %d", len(cfg), len(new)+4) + } + if d := fmt.Sprint(cfg["hostid"].(map[string]any)["description"]); !strings.Contains(d, "check of the pool's import at the next boot") { + t.Errorf("hostid's description does not say the change needs a check of the pool's import: %s", d) + } +} + +// §9 (A-15, A-23), §12, §14 (A-2, A-7), §15: the config file is `${config-json}` and not trusted; the options file, ZED's folder, rc, +// functions, handlers and drop-in each under their condition; no link and no resource or template for /etc/hostid. +func TestTheManifestsFilesAreAsSections9To15Say(t *testing.T) { + _, byID, _ := manifestOf(t) + c := byID["config"] + if c["path"] != ConfigFile || c["mode"] != "0644" || c["trusted"] != false || c["owner"] != nil || strings.TrimSpace(templateText(t, c)) != "${config-json}" { + t.Errorf("the config file: %v", c) + } + if byID["scrub-config"] != nil { + t.Errorf("scrub-config is gone: the node-engine removes the old file as a resource that left") + } + zedFiles := map[string]string{"zed-rc": "/etc/zfs/mesh-zed.d/zed.rc", "zed-functions": "/etc/zfs/mesh-zed.d/zed-functions.sh", + "zed-handler-all-syslog": "/etc/zfs/mesh-zed.d/all-syslog.sh", "zed-handler-scrub-after-resilver": "/etc/zfs/mesh-zed.d/resilver_finish-start-scrub.sh", + "zed-drop-in": "/etc/systemd/system/zfs-zed.service.d/mesh.conf"} + for id, path := range zedFiles { + r := byID[id] + want := "zed=on" + mode := "0644" + switch id { + case "zed-handler-all-syslog": + want, mode = "zed-handlers=all-syslog", "0755" + case "zed-handler-scrub-after-resilver": + want, mode = "zed-handlers=resilver_finish-start-scrub", "0755" + } + if r == nil || r["type"] != "file" || r["path"] != path || conditions(r) != want || r["mode"] != mode { + t.Errorf("%s: %v", id, r) + } + } + if d := byID["zed-dir"]; d["type"] != "directory" || d["path"] != "/etc/zfs/mesh-zed.d" || d["unknown"] != "module" || d["mode"] != "0755" || conditions(d) != "zed=on" { + t.Errorf("the folder: %v", d) + } + if o := byID["options"]; o["path"] != "/etc/modprobe.d/zfs.conf" || o["mode"] != "0644" || conditions(o) != "arc-max,arc-min,module-options" { + t.Errorf("the options file: %v", o) + } + if d := byID["run-dir"]; d["path"] != "/run/zfs-tools" || d["unknown"] != "program" || filepath.Dir(ApplyLock) != d["path"] { + t.Errorf("the lock's folder: %v", d) + } + for id, r := range byID { + if r["type"] == "symlink" || r["type"] == "link" || r["target"] != nil { + t.Errorf("%s is a link: the mesh creates none", id) + } + if r["path"] == "/etc/hostid" { + t.Errorf("%s places /etc/hostid: the apply writes it with ZFS's own writer", id) + } + if r["type"] == "file" && strings.Contains(templateText(t, r), "/etc/hostid") { + t.Errorf("%s's template names /etc/hostid", id) + } + } +} + +// §6, §13 (A-3), §18 (A-16): the run's resource precedes the resources that stop the scrub timers, with its schedule; +// each unit has the state and condition §13 lists; the trim timers are held stopped under none, weekly and monthly; the apply is +// run-once and starts again on the config file's change. +func TestTheManifestsUnitsAndTheRunAreAsSections6And13Say(t *testing.T) { + _, byID, order := manifestOf(t) + run := byID["run"] + if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || strings.Contains(fmt.Sprint(run["schedule"]), "/") { + t.Errorf("the scheduled run: %v", run) + } + if args, _ := run["run"].([]any); len(args) != 2 || args[1] != "run" { + t.Errorf("the run's command: %v", run["run"]) + } + if RunTimer != "zfs-mesh-run.timer" || RunService != "zfs-mesh-run.service" { + t.Errorf("the tools read %s and %s", RunTimer, RunService) + } + for _, id := range []string{"distribution-weekly", "distribution-monthly", "distribution-trim-weekly", "distribution-trim-monthly"} { + if r := byID[id]; r["state"] != "stopped" || r["boot"] != "disabled" || order["run"] > order[id] { + t.Errorf("%s: %v (the run is at %d, it at %d)", id, r, order["run"], order[id]) + } + } + for c, unit := range map[string]string{"distribution-weekly": "zfs-scrub-weekly@${config:pool}.timer", "distribution-monthly": "zfs-scrub-monthly@${config:pool}.timer", + "distribution-trim-weekly": "zfs-trim-weekly@${config:pool}.timer", "distribution-trim-monthly": "zfs-trim-monthly@${config:pool}.timer"} { + want := "" + if strings.Contains(c, "trim") { + want = "trim-schedule=none,trim-schedule=weekly,trim-schedule=monthly" + } + if byID[c]["unit"] != unit || conditions(byID[c]) != want { + t.Errorf("%s: %v", c, byID[c]) + } + } + units := map[string][3]string{"import-cache": {"zfs-import-cache.service", "running", "enabled"}, "mount": {"zfs-mount.service", "running", "enabled"}, + "target": {"zfs.target", "running", "enabled"}, "import-scan": {"zfs-import-scan.service", "", ""}, "share": {"zfs-share.service", "", ""}, + "volume-wait": {"zfs-volume-wait.service", "", ""}, "import-target": {"zfs-import.target", "", ""}} + for id, u := range units { + r := byID[id] + got := [3]string{fmt.Sprint(r["unit"]), fmt.Sprint(orNone(r["state"])), fmt.Sprint(orNone(r["boot"]))} + if r == nil || got != u || conditions(r) != "" { + t.Errorf("%s: %v, want %v", id, got, u) + } + } + on, off := byID["zed-on"], byID["zed-off"] + if on["unit"] != "zfs-zed.service" || on["state"] != "running" || on["boot"] != "enabled" || conditions(on) != "zed=on" || + off["unit"] != "zfs-zed.service" || off["state"] != "stopped" || off["boot"] != "disabled" || conditions(off) != "zed=off" { + t.Errorf("zed: %v %v", on, off) + } + if got := fmt.Sprint(on["restart-on"]); !strings.Contains(got, "zed-rc") || !strings.Contains(got, "zed-drop-in") || !strings.Contains(got, "zed-handler-all-syslog") { + t.Errorf("zed restarts on %v", got) + } + a := byID["apply"] + if a["type"] != "process" || a["name"] != "zfs-config-apply" || a["run-once"] != true || fmt.Sprint(a["restart-on"]) != "[config]" || a["schedule"] != nil || a["user"] != nil { + t.Errorf("the apply is run-once and starts on the config file's change: %v", a) + } + if order["config"] > order["apply"] || order["utils"] > order["config"] || order["options"] > order["apply"] || order["apply"] > order["run"] { + t.Errorf("the order: packages, config file and options before the apply, the apply before the run: %v", order) + } + for _, r := range byID { + if p, _ := r["path"].(string); strings.HasPrefix(p, "/etc/systemd/system/") && !strings.HasPrefix(p, "/etc/systemd/system/zfs-zed.service.d") { + t.Errorf("the module writes a unit file of its own: %s", p) + } + } +} + +func orNone(v any) any { + if v == nil { + return "" + } + return v +} + +// ADR 0240: zfs_health on the watch, which runs as the operator account; the follower judged by its unit. +// The scheduled run is not long-running and declares none. +func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { + _, byID, _ := manifestOf(t) w := byID["watch"] if h, _ := w["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "zfs_health" || w["user"] != "${machine:account}" { t.Errorf("the watch runs as the operator account and carries zfs_health: %v", w) @@ -101,119 +289,13 @@ func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { if h, _ := e["health"].(map[string]any); h["kind"] != "unit" || e["user"] != nil { t.Errorf("the follower runs as root, judged by its unit: %v", e) } - if d := byID["state-dir"]; d["path"] != "/var/lib/zfs-tools" || d["owner"] != "${machine:account}" || - filepath.Dir(ErrorsFile) != d["path"] { + if d := byID["state-dir"]; d["path"] != "/var/lib/zfs-tools" || d["owner"] != "${machine:account}" || filepath.Dir(ErrorsFile) != d["path"] { t.Errorf("the state the account keeps: %v", d) } // The scheduled run's state is root's alone, apart from the account's (third review). if d := byID["scrub-state-dir"]; d["path"] != filepath.Dir(ServedFile) || d["mode"] != "0700" || d["owner"] != nil { t.Errorf("the state the run keeps: %v", d) } - if strings.Contains(raw, `"package"`) { - t.Errorf("the module installs no package: ZFS is the machine's, and its kernel module is built against its kernel") - } -} - -// One schedule per pool, and no setting in any unit's name but the distribution's timers', which the -// controller fills (mesh-controller #171) and checks. The module's own schedule is the node-engine's timer -// for the scheduled run, a fixed name; the day and time reach only the configuration, which the run checks. -// The run comes before the distribution's timers are stopped, so a run that cannot be installed stops the -// apply before the timer that scrubs the pool today is touched. -func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { - m, raw := readManifest(t) - byID := map[string]map[string]any{} - order := map[string]int{} - for i, r := range m.Resources { - byID[r["id"].(string)] = r - order[r["id"].(string)] = i - } - run := byID["run"] - args, _ := run["run"].([]any) - if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || - len(args) != 2 || args[1] != "run" { - t.Errorf("the scheduled run: %v", run) - } - // The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never - // starts (novox/hq issue 1979, on the home server 2026-11-12). A list of minutes is written as a calendar - // systemd takes ("*-*-* *:0,15,30,45:00"), so the schedule names its minutes and never a step. - if s, _ := run["schedule"].(string); strings.Contains(s, "/") { - t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s) - } - if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" { - t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"]) - } - for _, c := range []string{"weekly", "monthly"} { - d := byID["distribution-"+c] - if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" { - t.Errorf("the distribution's %s timer: %v", c, d) - } - if order["run"] > order["distribution-"+c] { - t.Errorf("the distribution's %s timer is stopped before the run is installed", c) - } - } - for _, r := range m.Resources { - p, _ := r["path"].(string) - if strings.HasPrefix(p, "/etc/systemd/") { - t.Errorf("the module writes a unit file of its own: %s", p) - } - } - // The configuration's text is its template's (novox/hq ADR 0349 §2): the manifest itself names none of them. - conf := templateText(t, byID["scrub-config"]) - if strings.Contains(raw, "scrub-day}") || !strings.Contains(conf, "${config:scrub-day}") { - t.Errorf("the day reaches something other than the configuration") - } - if n := strings.Count(conf, "${config:scrub-day}") + strings.Count(conf, "${config:scrub-time}") + strings.Count(conf, "${config:scrub-cadence}"); n != 3 { - t.Errorf("the day, time and cadence are written %d times; only into the configuration", n) - } - var declared struct { - Config map[string]map[string]any `json:"config"` - } - json.Unmarshal([]byte(raw), &declared) - if declared.Config["scrub-cadence"]["default"] != "weekly" || declared.Config["scrub-day"]["default"] != "Mon" || - declared.Config["scrub-time"]["default"] != "00:00" { - t.Errorf("defaults: %v", declared.Config) - } - if p := declared.Config["scrub-pool"]; p["required"] != true || p["default"] != nil { - t.Errorf("the pool is the operator's value: required, with no default (ADR 0112, ADR 0348 §4): %v", p) - } -} - -// The home server, composed as the controller fills its settings (pool storage, cadence monthly, the -// operator's choice): the units the node-engine is sent and the configuration the run reads. -func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { - m, _ := readManifest(t) - fill := strings.NewReplacer("${config:scrub-pool}", "storage", "${config:scrub-cadence}", "monthly", - "${config:scrub-day}", "Mon", "${config:scrub-time}", "00:00") - var units []string - var conf string - for _, r := range m.Resources { - switch { - case r["type"] == "service": - units = append(units, fill.Replace(r["unit"].(string))+" "+r["state"].(string)+"/"+r["boot"].(string)) - case r["type"] == "process" && r["schedule"] != nil: - units = append(units, r["name"].(string)+".timer running/enabled (the node-engine's, every quarter of an hour)") - case r["id"] == "scrub-config": - conf = fill.Replace(templateText(t, r)) - } - } - want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)", - "zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"} - if strings.Join(units, "\n") != strings.Join(want, "\n") { - t.Errorf("units:\n%s", strings.Join(units, "\n")) - } - for _, u := range units { - t.Log(u) - } - _ = conf - cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000) - s := ScheduleOf(cfg) - if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" { - t.Fatalf("configuration: %v", refused) - } - t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04")) - if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) { - t.Errorf("next monthly slot %s, want the first Monday of November", next) - } } // Nothing but reading, a snapshot and a scrub's start, pause or stop. Every command the bundle runs goes diff --git a/modules/zfs/cmd/zfs-tools/templates_test.go b/modules/zfs/cmd/zfs-tools/templates_test.go index 7d0718b57..8b85a438c 100644 --- a/modules/zfs/cmd/zfs-tools/templates_test.go +++ b/modules/zfs/cmd/zfs-tools/templates_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -62,3 +63,34 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { t.Errorf("the build packs files/ into %d archives, not one", packs) } } + +// §9, §12, §14 (A-15), §31: the three templates. The config file is the module's values as one JSON object; the options file holds +// config-lines for the ARC (in bytes) and the options; ZED's rc holds none of the package's notification or mail lines, and the +// scrub-after-resilver variable only its handler reads. +func TestTheConfigOptionsAndZedTemplates(t *testing.T) { + _, byID, _ := manifestOf(t) + if got := strings.TrimSpace(templateText(t, byID["config"])); got != "${config-json}" { + t.Errorf("the config file: %q", got) + } + opts := templateText(t, byID["options"]) + for _, want := range []string{"${config-lines:arc-max:options zfs zfs_arc_max=:mib-bytes}", "${config-lines:arc-min:options zfs zfs_arc_min=:mib-bytes}", + "${config-lines:module-options:options zfs }"} { + if !strings.Contains(opts, want) { + t.Errorf("the options file lacks %s", want) + } + } + rc := templateText(t, byID["zed-rc"]) + for _, bad := range []string{"ZED_NOTIFY", "ZED_EMAIL", "PUSHBULLET", "SLACK", "PUSHOVER", "NTFY", "GOTIFY"} { + for _, l := range strings.Split(rc, "\n") { + if strings.Contains(l, bad) && !strings.HasPrefix(l, "#") { + t.Errorf("zed.rc holds %s: %s", bad, l) + } + } + } + if !strings.Contains(rc, "\nZED_SCRUB_AFTER_RESILVER=1\n") { + t.Errorf("zed.rc does not enable the scrub after a resilver for its handler") + } + if drop := templateText(t, byID["zed-drop-in"]); !strings.Contains(drop, "ExecStart=/usr/bin/zed -F -d /etc/zfs/mesh-zed.d") { + t.Errorf("the drop-in does not run ZED from the module's folder: %s", drop) + } +} diff --git a/modules/zfs/files/all-syslog.sh b/modules/zfs/files/all-syslog.sh new file mode 100644 index 000000000..5f601144a --- /dev/null +++ b/modules/zfs/files/all-syslog.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +# +# Log the zevent via syslog. +# + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +zed_exit_if_ignoring_this_event + +# build a string of name=value pairs for this event +msg="eid=${ZEVENT_EID} class=${ZEVENT_SUBCLASS}" + +if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then + [ -n "${ZEVENT_POOL_GUID}" ] && msg="${msg} pool_guid=${ZEVENT_POOL_GUID}" + [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}" +else + [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'" + [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}" +fi + +# log pool state if state is anything other than 'ACTIVE' +[ -n "${ZEVENT_POOL_STATE_STR}" ] && [ "$ZEVENT_POOL_STATE" -ne 0 ] && \ + msg="${msg} pool_state=${ZEVENT_POOL_STATE_STR}" + +# Log the following payload nvpairs if they are present +[ -n "${ZEVENT_VDEV_STATE_STR}" ] && msg="${msg} vdev_state=${ZEVENT_VDEV_STATE_STR}" +[ -n "${ZEVENT_CKSUM_ALGORITHM}" ] && msg="${msg} algorithm=${ZEVENT_CKSUM_ALGORITHM}" +[ -n "${ZEVENT_ZIO_SIZE}" ] && msg="${msg} size=${ZEVENT_ZIO_SIZE}" +[ -n "${ZEVENT_ZIO_OFFSET}" ] && msg="${msg} offset=${ZEVENT_ZIO_OFFSET}" +[ -n "${ZEVENT_ZIO_PRIORITY}" ] && msg="${msg} priority=${ZEVENT_ZIO_PRIORITY}" +[ -n "${ZEVENT_ZIO_ERR}" ] && msg="${msg} err=${ZEVENT_ZIO_ERR}" +[ -n "${ZEVENT_ZIO_FLAGS}" ] && msg="${msg} flags=$(printf '0x%x' "${ZEVENT_ZIO_FLAGS}")" + +# log delays that are >= 10 milisec +[ -n "${ZEVENT_ZIO_DELAY}" ] && [ "$ZEVENT_ZIO_DELAY" -gt 10000000 ] && \ + msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms" + +# list the bookmark data together +# shellcheck disable=SC2153 +[ -n "${ZEVENT_ZIO_OBJSET}" ] && \ + msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}" + +zed_log_msg "${msg}" + +exit 0 diff --git a/modules/zfs/files/config.json b/modules/zfs/files/config.json new file mode 100644 index 000000000..0984e89a2 --- /dev/null +++ b/modules/zfs/files/config.json @@ -0,0 +1 @@ +${config-json} diff --git a/modules/zfs/files/resilver_finish-start-scrub.sh b/modules/zfs/files/resilver_finish-start-scrub.sh new file mode 100644 index 000000000..cafce6fde --- /dev/null +++ b/modules/zfs/files/resilver_finish-start-scrub.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# resilver_finish-start-scrub.sh +# Run a scrub after a resilver +# +# Exit codes: +# 1: Internal error +# 2: Script wasn't enabled in zed.rc +# 3: Scrubs are automatically started for sequential resilvers +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2 +[ "${ZEVENT_RESILVER_TYPE}" != "sequential" ] || exit 3 +[ -n "${ZEVENT_POOL}" ] || exit 1 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 1 +zed_check_cmd "${ZPOOL}" || exit 1 + +zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}" +"${ZPOOL}" scrub "${ZEVENT_POOL}" diff --git a/modules/zfs/files/scrub.conf b/modules/zfs/files/scrub.conf deleted file mode 100644 index daccbd93f..000000000 --- a/modules/zfs/files/scrub.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Written by the mesh (module zfs) from this machine's assignment. Replaced on every push; change the -# settings scrub-pool, scrub-cadence, scrub-day and scrub-time, never this file. The module's tools read it. -pool=${config:scrub-pool} -cadence=${config:scrub-cadence} -day=${config:scrub-day} -time=${config:scrub-time} diff --git a/modules/zfs/files/zed-functions.sh b/modules/zfs/files/zed-functions.sh new file mode 100644 index 000000000..78d8f658d --- /dev/null +++ b/modules/zfs/files/zed-functions.sh @@ -0,0 +1,825 @@ +#!/bin/sh +# shellcheck disable=SC2154,SC3043 +# zed-functions.sh +# +# ZED helper functions for use in ZEDLETs + + +# Variable Defaults +# +: "${ZED_LOCKDIR:="/var/lock"}" +: "${ZED_NOTIFY_INTERVAL_SECS:=3600}" +: "${ZED_NOTIFY_VERBOSE:=0}" +: "${ZED_RUNDIR:="/var/run"}" +: "${ZED_SYSLOG_PRIORITY:="daemon.notice"}" +: "${ZED_SYSLOG_TAG:="zed"}" + +ZED_FLOCK_FD=8 + + +# zed_check_cmd (cmd, ...) +# +# For each argument given, search PATH for the executable command [cmd]. +# Log a message if [cmd] is not found. +# +# Arguments +# cmd: name of executable command for which to search +# +# Return +# 0 if all commands are found in PATH and are executable +# n for a count of the command executables that are not found +# +zed_check_cmd() +{ + local cmd + local rv=0 + + for cmd; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + zed_log_err "\"${cmd}\" not installed" + rv=$((rv + 1)) + fi + done + return "${rv}" +} + + +# zed_log_msg (msg, ...) +# +# Write all argument strings to the system log. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# +# Return +# nothing +# +zed_log_msg() +{ + logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "$@" +} + + +# zed_log_err (msg, ...) +# +# Write an error message to the system log. This message will contain the +# script name, EID, and all argument strings. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# ZEVENT_EID +# +# Return +# nothing +# +zed_log_err() +{ + zed_log_msg "error: ${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" +} + + +# zed_lock (lockfile, [fd]) +# +# Obtain an exclusive (write) lock on [lockfile]. If the lock cannot be +# immediately acquired, wait until it becomes available. +# +# Every zed_lock() must be paired with a corresponding zed_unlock(). +# +# By default, flock-style locks associate the lockfile with file descriptor 8. +# The bash manpage warns that file descriptors >9 should be used with care as +# they may conflict with file descriptors used internally by the shell. File +# descriptor 9 is reserved for zed_rate_limit(). If concurrent locks are held +# within the same process, they must use different file descriptors (preferably +# decrementing from 8); otherwise, obtaining a new lock with a given file +# descriptor will release the previous lock associated with that descriptor. +# +# Arguments +# lockfile: pathname of the lock file; the lock will be stored in +# ZED_LOCKDIR unless the pathname contains a "/". +# fd: integer for the file descriptor used by flock (OPTIONAL unless holding +# concurrent locks) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_lock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local umask_bak + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + umask_bak="$(umask)" + umask 077 + + # Obtain a lock on the file bound to the given file descriptor. + # + eval "exec ${fd}>> '${lockfile}'" + if ! err="$(flock --exclusive "${fd}" 2>&1)"; then + zed_log_err "failed to lock \"${lockfile}\": ${err}" + fi + + umask "${umask_bak}" +} + + +# zed_unlock (lockfile, [fd]) +# +# Release the lock on [lockfile]. +# +# Arguments +# lockfile: pathname of the lock file +# fd: integer for the file descriptor used by flock (must match the file +# descriptor passed to the zed_lock function call) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_unlock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + # Release the lock and close the file descriptor. + if ! err="$(flock --unlock "${fd}" 2>&1)"; then + zed_log_err "failed to unlock \"${lockfile}\": ${err}" + fi + eval "exec ${fd}>&-" +} + + +# zed_notify (subject, pathname) +# +# Send a notification via all available methods. +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Return +# 0: notification succeeded via at least one method +# 1: notification failed +# 2: no notification methods configured +# +zed_notify() +{ + local subject="$1" + local pathname="$2" + local num_success=0 + local num_failure=0 + + zed_notify_email "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushbullet "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_slack_webhook "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushover "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_ntfy "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_gotify "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + [ "${num_success}" -gt 0 ] && return 0 + [ "${num_failure}" -gt 0 ] && return 1 + return 2 +} + + +# zed_notify_email (subject, pathname) +# +# Send a notification via email to the address specified by ZED_EMAIL_ADDR. +# +# Requires the mail executable to be installed in the standard PATH, or +# ZED_EMAIL_PROG to be defined with the pathname of an executable capable of +# reading a message body from stdin. +# +# Command-line options to the mail executable can be specified in +# ZED_EMAIL_OPTS. This undergoes the following keyword substitutions: +# - @ADDRESS@ is replaced with the space-delimited recipient email address(es) +# - @SUBJECT@ is replaced with the notification subject +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_EMAIL_PROG +# ZED_EMAIL_OPTS +# ZED_EMAIL_ADDR +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_email() +{ + local subject="${1:-"ZED notification"}" + local pathname="${2:-"/dev/null"}" + + : "${ZED_EMAIL_PROG:="mail"}" + : "${ZED_EMAIL_OPTS:="-s '@SUBJECT@' @ADDRESS@"}" + + # For backward compatibility with ZED_EMAIL. + if [ -n "${ZED_EMAIL}" ] && [ -z "${ZED_EMAIL_ADDR}" ]; then + ZED_EMAIL_ADDR="${ZED_EMAIL}" + fi + [ -n "${ZED_EMAIL_ADDR}" ] || return 2 + + zed_check_cmd "${ZED_EMAIL_PROG}" || return 1 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err \ + "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\"" + return 1 + fi + + # construct cmdline options + ZED_EMAIL_OPTS_PARSED="$(echo "${ZED_EMAIL_OPTS}" \ + | sed -e "s/@ADDRESS@/${ZED_EMAIL_ADDR}/g" \ + -e "s/@SUBJECT@/${subject}/g")" + + # pipe message to email prog + # shellcheck disable=SC2086,SC2248 + { + # no subject passed as option? + if [ "${ZED_EMAIL_OPTS%@SUBJECT@*}" = "${ZED_EMAIL_OPTS}" ] ; then + # inject subject header + printf "Subject: %s\n" "${subject}" + # The following empty line is needed to separate the header from the + # body of the message. Otherwise programs like sendmail will skip + # everything up to the first empty line (or wont send an email at + # all) and will still exit with exit code 0 + printf "\n" + fi + # output message + cat "${pathname}" + } | + eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS_PARSED} >/dev/null 2>&1 + rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}" + return 1 + fi + return 0 +} + + +# zed_notify_pushbullet (subject, pathname) +# +# Send a notification via Pushbullet . +# The access token (ZED_PUSHBULLET_ACCESS_TOKEN) identifies this client to the +# Pushbullet server. The optional channel tag (ZED_PUSHBULLET_CHANNEL_TAG) is +# for pushing to notification feeds that can be subscribed to; if a channel is +# not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://docs.pushbullet.com/ +# https://www.pushbullet.com/security +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHBULLET_ACCESS_TOKEN +# ZED_PUSHBULLET_CHANNEL_TAG +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushbullet() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="https://api.pushbullet.com/v2/pushes" + + [ -n "${ZED_PUSHBULLET_ACCESS_TOKEN}" ] || return 2 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "pushbullet cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Push to a channel if one is configured. + # + [ -n "${ZED_PUSHBULLET_CHANNEL_TAG}" ] && msg_tag="$(printf \ + '"channel_tag": "%s", ' "${ZED_PUSHBULLET_CHANNEL_TAG}")" + + # Construct the JSON message for pushing a note. + # + msg_json="$(printf '{%s"type": "note", "title": "%s", "body": "%s"}' \ + "${msg_tag}" "${subject}" "${msg_body}")" + + # Send the POST request and check for errors. + # + msg_out="$(curl -u "${ZED_PUSHBULLET_ACCESS_TOKEN}:" -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushbullet \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_slack_webhook (subject, pathname) +# +# Notification via Slack Webhook . +# The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the +# Slack channel. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://api.slack.com/incoming-webhooks +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_SLACK_WEBHOOK_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_slack_webhook() +{ + [ -n "${ZED_SLACK_WEBHOOK_URL}" ] || return 2 + + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="${ZED_SLACK_WEBHOOK_URL}" + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "slack webhook cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Construct the JSON message for posting. + # shellcheck disable=SC2016 + # + msg_json="$(printf '{"text": "*%s*\\n```%s```"}' "${subject}" "${msg_body}" )" + + # Send the POST request and check for errors. + # + msg_out="$(curl -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "slack webhook \"${msg_err}"\" + return 1 + fi + return 0 +} + +# zed_notify_pushover (subject, pathname) +# +# Send a notification via Pushover . +# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the +# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or +# group to which the notification will be sent. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://pushover.net/api +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHOVER_TOKEN +# ZED_PUSHOVER_USER +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushover() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + local url="https://api.pushover.net/1/messages.json" + + [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2 + + if [ ! -r "${pathname}" ]; then + zed_log_err "pushover cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + msg_out="$( \ + curl \ + --form-string "token=${ZED_PUSHOVER_TOKEN}" \ + --form-string "user=${ZED_PUSHOVER_USER}" \ + --form-string "message=${msg_body}" \ + --form-string "title=${subject}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushover \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_ntfy (subject, pathname) +# +# Send a notification via Ntfy.sh . +# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification +# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the +# self-hosted or provided hosted ntfy service location. The ntfy access token +# (ZED_NTFY_ACCESS_TOKEN) reprsents an +# access token that could be used if a topic is read/write protected. If a +# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://docs.ntfy.sh +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_NTFY_TOPIC +# ZED_NTFY_ACCESS_TOKEN (OPTIONAL) +# ZED_NTFY_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_ntfy() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_NTFY_TOPIC}" ] || return 2 + local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "ntfy cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then + msg_out="$( \ + curl \ + -u ":${ZED_NTFY_ACCESS_TOKEN}" \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "ntfy \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_gotify (subject, pathname) +# +# Send a notification via Gotify . +# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location. +# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a +# Gotify application token which is associated with a message. +# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the +# default or configured priority at the Gotify server for the application. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://gotify.net/docs/index +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_GOTIFY_URL +# ZED_GOTIFY_APPTOKEN +# ZED_GOTIFY_PRIORITY +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_gotify() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2 + local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "gotify cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + --form-string "priority=${ZED_GOTIFY_PRIORITY}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "gotify \"${msg_err}"\" + return 1 + fi + return 0 +} + + + +# zed_rate_limit (tag, [interval]) +# +# Check whether an event of a given type [tag] has already occurred within the +# last [interval] seconds. +# +# This function obtains a lock on the statefile using file descriptor 9. +# +# Arguments +# tag: arbitrary string for grouping related events to rate-limit +# interval: time interval in seconds (OPTIONAL) +# +# Globals +# ZED_NOTIFY_INTERVAL_SECS +# ZED_RUNDIR +# +# Return +# 0 if the event should be processed +# 1 if the event should be dropped +# +# State File Format +# time;tag +# +zed_rate_limit() +{ + local tag="$1" + local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}" + local lockfile="zed.zedlet.state.lock" + local lockfile_fd=9 + local statefile="${ZED_RUNDIR}/zed.zedlet.state" + local time_now + local time_prev + local umask_bak + local rv=0 + + [ -n "${tag}" ] || return 0 + + zed_lock "${lockfile}" "${lockfile_fd}" + time_now="$(date +%s)" + time_prev="$(grep -E "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + | tail -1 | cut -d\; -f1)" + + if [ -n "${time_prev}" ] \ + && [ "$((time_now - time_prev))" -lt "${interval}" ]; then + rv=1 + else + umask_bak="$(umask)" + umask 077 + grep -E -v "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + > "${statefile}.$$" + echo "${time_now};${tag}" >> "${statefile}.$$" + mv -f "${statefile}.$$" "${statefile}" + umask "${umask_bak}" + fi + + zed_unlock "${lockfile}" "${lockfile_fd}" + return "${rv}" +} + + +# zed_guid_to_pool (guid) +# +# Convert a pool GUID into its pool name (like "tank") +# Arguments +# guid: pool GUID (decimal or hex) +# +# Return +# Pool name +# +zed_guid_to_pool() +{ + if [ -z "$1" ] ; then + return + fi + + guid="$(printf "%u" "$1")" + $ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}' +} + +# zed_exit_if_ignoring_this_event +# +# Exit the script if we should ignore this event, as determined by +# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc. +# This function assumes you've imported the normal zed variables. +zed_exit_if_ignoring_this_event() +{ + if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_INCLUDE});; + *) exit 0;; + esac" + elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;; + *);; + esac" + fi +} diff --git a/modules/zfs/files/zed-mesh.conf b/modules/zfs/files/zed-mesh.conf new file mode 100644 index 000000000..196be4669 --- /dev/null +++ b/modules/zfs/files/zed-mesh.conf @@ -0,0 +1,4 @@ +# Written by the mesh (module zfs). ZED runs from the module's own folder, which holds no link. +[Service] +ExecStart= +ExecStart=/usr/bin/zed -F -d /etc/zfs/mesh-zed.d diff --git a/modules/zfs/files/zed.rc b/modules/zfs/files/zed.rc new file mode 100644 index 000000000..562b9f594 --- /dev/null +++ b/modules/zfs/files/zed.rc @@ -0,0 +1,7 @@ +# Written by the mesh (module zfs). Replaced on every push; change the settings zed and zed-handlers, never this file. +# It holds the package's own variables but its notification and mail lines: a pool's fault reaches the operator once, +# from the event follower's finding, never from a ZED handler. +ZED_USE_ENCLOSURE_LEDS=1 +ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event" +# Read only by the scrub-after-resilver handler, which is placed only while zed-handlers names it. +ZED_SCRUB_AFTER_RESILVER=1 diff --git a/modules/zfs/files/zfs.conf b/modules/zfs/files/zfs.conf new file mode 100644 index 000000000..edc788e72 --- /dev/null +++ b/modules/zfs/files/zfs.conf @@ -0,0 +1,5 @@ +# Written by the mesh (module zfs) from this machine's assignment. Replaced on every push; change the +# settings arc-max, arc-min and module-options, never this file. ZFS reads it when its kernel module loads. +${config-lines:arc-max:options zfs zfs_arc_max=:mib-bytes} +${config-lines:arc-min:options zfs zfs_arc_min=:mib-bytes} +${config-lines:module-options:options zfs } diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 200e26364..2f119f5ab 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -2,52 +2,151 @@ "module": "zfs", "version": "1", "capabilities": [ + "package-manager", "service-manager" ], "emits": [ "snapshot.taken", "scrub.requested" ], - "config": { - "scrub-pool": { - "type": "string", - "description": "The name of the ZFS pool the module scrubs; the operator's, with no default.", - "trusted": true, - "required": true - }, - "scrub-cadence": { - "type": "string", - "description": "How often the pool named by scrub-pool is scrubbed: weekly, or monthly on the first scrub-day of each month. Weekly is what the distribution's timer did on the home server before the module, so taking it over moves nothing until the operator chooses.", - "enum": [ - "weekly", - "monthly" - ], - "trusted": true, - "default": "weekly" - }, - "scrub-day": { - "type": "string", - "description": "The weekday the scrub starts, Mon to Sun; for a monthly scrub, the first such day of the month. Checked by the scheduled run, which refuses any other value and runs nothing; Monday is when the distribution's weekly timer fired.", - "enum": [ - "Mon", - "Tue", - "Wed", - "Thu", - "Fri", - "Sat", - "Sun" - ], - "trusted": true, - "default": "Mon" - }, - "scrub-time": { - "type": "string", - "description": "The time the scrub starts, HH:MM from 00:00 to 23:59 in the machine's time zone; the scheduled run checks it every quarter of an hour, so the scrub starts within 15 minutes of it. Midnight is when the distribution's weekly timer fired.", - "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", - "trusted": true, - "default": "00:00" - } + "config": { + "pool": { + "type": "string", + "description": "The name of the ZFS pool the module owns: scrubs, trims, takes scheduled ZFS snapshots of and converges; the operator's, with no default.", + "trusted": true, + "required": true, + "renamed-from": "scrub-pool" }, + "scrub-cadence": { + "type": "string", + "description": "How often the pool named by pool is scrubbed: weekly, or monthly on the first scrub-day of each month. Weekly is what the distribution's timer did on the home server before the module, so taking it over moves nothing until the operator chooses.", + "enum": [ + "weekly", + "monthly" + ], + "trusted": true, + "default": "weekly" + }, + "scrub-day": { + "type": "string", + "description": "The weekday the scrub starts, Mon to Sun; for a monthly scrub, the first such day of the month. Checked by the scheduled run, which refuses any other value and runs nothing; Monday is when the distribution's weekly timer fired.", + "enum": [ + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", + "Sun" + ], + "trusted": true, + "default": "Mon" + }, + "scrub-time": { + "type": "string", + "description": "The time the scrub starts, HH:MM from 00:00 to 23:59 in the machine's time zone; the scheduled run checks it every quarter of an hour, so the scrub starts within 15 minutes of it. Midnight is when the distribution's weekly timer fired.", + "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", + "trusted": true, + "default": "00:00" + }, + "arc-max": { + "type": "number", + "description": "The most ZFS's read cache in memory (the ARC) may hold, in whole MiB: at least 64 and below the node's memory. Written live and into the options file. Not managed until set.", + "minimum": 64, + "trusted": false + }, + "arc-min": { + "type": "number", + "description": "The least the ARC keeps, in whole MiB, not above arc-max. Written live and into the options file. Not managed until set.", + "minimum": 0, + "trusted": false + }, + "module-options": { + "type": "list", + "description": "Kernel module parameters, each name=digits, from l2arc_write_max, l2arc_write_boost, l2arc_headroom, l2arc_noprefetch, l2arc_rebuild_enabled, zfs_txg_timeout, zfs_dirty_data_max, zfs_scan_vdev_limit, zfs_scrub_min_time_ms, zfs_resilver_min_time_ms and zfs_arc_sys_free (arc-max and arc-min hold the ARC's). Written into the options file; a live one is also written live, the others take effect at the next load of the kernel module. An empty list is not managed.", + "trusted": true + }, + "hostid": { + "type": "string", + "description": "The node's ZFS id: eight hexadecimal digits, not all zero, written to /etc/hostid with ZFS's own writer only when the pool's label holds 0 or this value. Changing it needs a check of the pool's import at the next boot; the module never reboots. Not managed until set.", + "pattern": "^[0-9a-fA-F]{8}$", + "trusted": true + }, + "zed": { + "type": "string", + "description": "Whether the ZFS event daemon runs from the module's own folder: on or off. Not managed until set.", + "enum": [ + "on", + "off" + ], + "trusted": false + }, + "zed-handlers": { + "type": "list", + "description": "The event handlers placed in the module's folder: all-syslog (the journal handler) and resilver_finish-start-scrub (a scrub after a resilver). Empty is not managed.", + "trusted": true + }, + "trim-schedule": { + "type": "string", + "description": "How often the pool is trimmed by the scheduled run: none, weekly, or monthly on the first trim-day of each month. Any set value holds the distribution's trim timers stopped. Needs trim-day and trim-time for weekly and monthly. Not managed until set.", + "enum": [ + "none", + "weekly", + "monthly" + ], + "trusted": false + }, + "trim-day": { + "type": "string", + "description": "The weekday the trim starts, Mon to Sun; for monthly, the first such day of the month. Not managed until set.", + "enum": [ + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", + "Sun" + ], + "trusted": false + }, + "trim-time": { + "type": "string", + "description": "The time the trim starts, HH:MM in the machine's time zone, within 15 minutes. Not managed until set.", + "pattern": "([01][0-9]|2[0-3]):[0-5][0-9]", + "trusted": false + }, + "cache-devices": { + "type": "list", + "description": "The pool's cache devices, by their names under /dev/disk/by-id: letters, digits, -, _, . and : only. A kernel-named member is re-added by its stable name; an empty list removes nothing and is not managed.", + "trusted": true + }, + "log-devices": { + "type": "list", + "description": "The pool's log devices, by their names under /dev/disk/by-id. An empty list removes nothing and is not managed.", + "trusted": true + }, + "log-mirror": { + "type": "boolean", + "description": "Whether log-devices are one mirror: true needs exactly two names, added as one mirror; otherwise each is a log device of its own. Not managed until set.", + "trusted": true + }, + "pool-properties": { + "type": "object", + "description": "The pool's properties: autotrim, autoexpand, autoreplace (on, off or default) and failmode (wait, continue, panic or default). Not managed until set.", + "trusted": true + }, + "dataset-properties": { + "type": "object", + "description": "Per dataset of the pool, its properties: compression, atime, relatime, recordsize, primarycache, secondarycache, sync, logbias, quota and reservation, each in its own form; never sharenfs or sharesmb. Not managed until set.", + "trusted": true + }, + "snapshots": { + "type": "object", + "description": "The scheduled ZFS snapshots of every filesystem of the pool: {\"none\": true}, or keep counts hourly, daily, weekly and monthly, each a whole number from 0 to 168; a count of 0 takes none of that kind and removes none. Only snapshots the module took, named mesh-- and recorded in its ledger, are ever removed. Not managed until set.", + "trusted": true + } + }, "tools": [ "zfs_datasets", "zfs_snapshots", @@ -129,6 +228,224 @@ ] }, "resources": [ + { + "id": "utils", + "type": "package", + "package": "zfs-utils" + }, + { + "id": "dkms", + "type": "package", + "package": "zfs-dkms" + }, + { + "id": "state-dir", + "type": "directory", + "path": "/var/lib/zfs-tools", + "mode": "0755", + "owner": "${machine:account}", + "unknown": "program" + }, + { + "id": "cursor-dir", + "type": "directory", + "path": "/run/zfs-events", + "mode": "0700", + "unknown": "program" + }, + { + "id": "run-dir", + "type": "directory", + "path": "/run/zfs-tools", + "mode": "0755", + "unknown": "program" + }, + { + "id": "config-dir", + "type": "directory", + "path": "/etc/zfs-tools", + "mode": "0755", + "unknown": "program" + }, + { + "id": "scrub-state-dir", + "type": "directory", + "path": "/var/lib/zfs-scrub", + "mode": "0700", + "unknown": "program" + }, + { + "id": "config", + "type": "file", + "path": "/etc/zfs-tools/config.json", + "mode": "0644", + "template": "files/config.json", + "trusted": false + }, + { + "id": "options", + "type": "file", + "path": "/etc/modprobe.d/zfs.conf", + "mode": "0644", + "template": "files/zfs.conf", + "if-config": [ + "arc-max", + "arc-min", + "module-options" + ] + }, + { + "id": "zed-dir", + "type": "directory", + "path": "/etc/zfs/mesh-zed.d", + "mode": "0755", + "unknown": "module", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-rc", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/zed.rc", + "mode": "0644", + "template": "files/zed.rc", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-functions", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/zed-functions.sh", + "mode": "0644", + "template": "files/zed-functions.sh", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-handler-all-syslog", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/all-syslog.sh", + "mode": "0755", + "template": "files/all-syslog.sh", + "if-config": [ + "zed-handlers=all-syslog" + ] + }, + { + "id": "zed-handler-scrub-after-resilver", + "type": "file", + "path": "/etc/zfs/mesh-zed.d/resilver_finish-start-scrub.sh", + "mode": "0755", + "template": "files/resilver_finish-start-scrub.sh", + "if-config": [ + "zed-handlers=resilver_finish-start-scrub" + ] + }, + { + "id": "zed-drop-ins", + "type": "directory", + "path": "/etc/systemd/system/zfs-zed.service.d", + "mode": "0755", + "unknown": "program", + "if-config": [ + "zed=on" + ] + }, + { + "id": "zed-drop-in", + "type": "file", + "path": "/etc/systemd/system/zfs-zed.service.d/mesh.conf", + "mode": "0644", + "template": "files/zed-mesh.conf", + "if-config": [ + "zed=on" + ] + }, + { + "id": "import-cache", + "type": "service", + "unit": "zfs-import-cache.service", + "state": "running", + "boot": "enabled" + }, + { + "id": "mount", + "type": "service", + "unit": "zfs-mount.service", + "state": "running", + "boot": "enabled" + }, + { + "id": "target", + "type": "service", + "unit": "zfs.target", + "state": "running", + "boot": "enabled" + }, + { + "id": "import-scan", + "type": "service", + "unit": "zfs-import-scan.service" + }, + { + "id": "share", + "type": "service", + "unit": "zfs-share.service" + }, + { + "id": "volume-wait", + "type": "service", + "unit": "zfs-volume-wait.service" + }, + { + "id": "import-target", + "type": "service", + "unit": "zfs-import.target" + }, + { + "id": "zed-on", + "type": "service", + "unit": "zfs-zed.service", + "state": "running", + "boot": "enabled", + "if-config": [ + "zed=on" + ], + "restart-on": [ + "zed-rc", + "zed-functions", + "zed-handler-all-syslog", + "zed-handler-scrub-after-resilver", + "zed-drop-in" + ] + }, + { + "id": "zed-off", + "type": "service", + "unit": "zfs-zed.service", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "zed=off" + ] + }, + { + "id": "apply", + "type": "process", + "name": "zfs-config-apply", + "artifact": "tools", + "run": [ + "./zfs-tools", + "apply" + ], + "run-once": true, + "restart-on": [ + "config" + ] + }, { "id": "watch", "type": "process", @@ -148,21 +465,6 @@ "grace": "60s" } }, - { - "id": "state-dir", - "type": "directory", - "path": "/var/lib/zfs-tools", - "mode": "0755", - "owner": "${machine:account}", - "unknown": "program" - }, - { - "id": "cursor-dir", - "type": "directory", - "path": "/run/zfs-events", - "mode": "0700", - "unknown": "program" - }, { "id": "events", "type": "process", @@ -176,27 +478,6 @@ "kind": "unit" } }, - { - "id": "config-dir", - "type": "directory", - "path": "/etc/zfs-tools", - "mode": "0755", - "unknown": "program" - }, - { - "id": "scrub-config", - "type": "file", - "path": "/etc/zfs-tools/scrub.conf", - "mode": "0644", - "template": "files/scrub.conf" - }, - { - "id": "scrub-state-dir", - "type": "directory", - "path": "/var/lib/zfs-scrub", - "mode": "0700", - "unknown": "program" - }, { "id": "run", "type": "process", @@ -211,16 +492,40 @@ { "id": "distribution-weekly", "type": "service", - "unit": "zfs-scrub-weekly@${config:scrub-pool}.timer", + "unit": "zfs-scrub-weekly@${config:pool}.timer", "state": "stopped", "boot": "disabled" }, { "id": "distribution-monthly", "type": "service", - "unit": "zfs-scrub-monthly@${config:scrub-pool}.timer", + "unit": "zfs-scrub-monthly@${config:pool}.timer", "state": "stopped", "boot": "disabled" + }, + { + "id": "distribution-trim-weekly", + "type": "service", + "unit": "zfs-trim-weekly@${config:pool}.timer", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "trim-schedule=none", + "trim-schedule=weekly", + "trim-schedule=monthly" + ] + }, + { + "id": "distribution-trim-monthly", + "type": "service", + "unit": "zfs-trim-monthly@${config:pool}.timer", + "state": "stopped", + "boot": "disabled", + "if-config": [ + "trim-schedule=none", + "trim-schedule=weekly", + "trim-schedule=monthly" + ] } ], "build": { From 9116067e11ca7e4788e4dca513d18b3d5a8ed5f6 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:01:35 +0100 Subject: [PATCH 16/35] Z18 zfs-callsite: the call-site test pins every form of ADR 0679 whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renamed TestTheBundleRunsOnlyTheFormsOfADR0679: each root form is held to its whole argument list, no verb the decision never allows, zgenhostid only with the parsed hostid, a device only as cache or log. Builds: design 42 zfs-owns-zfs §19 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 78 ++++++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index c8e5f7a45..3c028f2a0 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -298,11 +298,10 @@ func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) { } } -// Nothing but reading, a snapshot and a scrub's start, pause or stop. Every command the bundle runs goes -// through ask, AsRoot, Escalated, Run, exec.Command or exec.CommandContext with an argument list from one -// of the functions below, each held to its program and verb; a call site that builds its own list fails -// this test. A new verb needs a decision first, and then a line here. -func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { +// §19, ADR 0679 §7, §17: every command the bundle runs goes through ask, AsRoot, Escalated, Run, exec.Command or +// exec.CommandContext with an argument list from one of the functions below, each held to its program, its verb and its whole +// form; a call site that builds its own list fails this test. A new verb needs a decision first, and then a line here. +func TestTheBundleRunsOnlyTheFormsOfADR0679(t *testing.T) { allowed := map[string][2]string{ "DatasetsArgv": {"zfs", "list"}, "SnapshotsArgv": {"zfs", "list"}, "PropertiesArgv": {"zfs", "get"}, "SnapshotArgv": {"zfs", "snapshot"}, @@ -339,6 +338,73 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { t.Errorf("%s runs %q, not %q", fn, argv[0], allowed[fn][1]) } } + // The whole form of each root form §19 pins, and never a forbidden verb or flag. + pinned := map[string][]string{ + "AddArgv cache": AddArgv("tank", "cache", false, "/dev/a", "/dev/b"), + "AddArgv log": AddArgv("tank", "log", true, "/dev/a", "/dev/b"), + "RemoveArgv": RemoveArgv("tank", "/dev/a"), + "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"), + "DatasetSetArgv": DatasetSetArgv("tank/a", "atime", "off"), + "DatasetInherit": DatasetInheritArgv("tank/a", "atime"), + "SnapshotArgv": SnapshotArgv("tank/a", "x"), + "SnapshotSetArgv": SnapshotSetArgv("tank/a@mesh-hourly-x", "hourly"), + "DestroyArgv": DestroyArgv("tank/a", "mesh-hourly-x"), + "TrimArgv": TrimArgv("tank"), + "WipefsArgv": WipefsArgv("/dev/a"), + "ZdbLabelArgv": ZdbLabelArgv("/dev/a"), + "ZdbConfigArgv": ZdbConfigArgv("tank"), + "ZgenhostidArgv": ZgenhostidArgv("0a0b0c0d"), + "UdevArgv": UdevArgv(), + } + forms := map[string]string{ + "AddArgv cache": "add -- tank cache /dev/a /dev/b", "AddArgv log": "add -- tank log mirror /dev/a /dev/b", "RemoveArgv": "remove -- tank /dev/a", + "PoolSetArgv": "set autotrim=on -- tank", "DatasetSetArgv": "set atime=off -- tank/a", "DatasetInherit": "inherit atime -- tank/a", + "SnapshotArgv": "snapshot -- tank/a@x", "SnapshotSetArgv": "set mesh:schedule=hourly -- tank/a@mesh-hourly-x", + "DestroyArgv": "destroy -- tank/a@mesh-hourly-x", "TrimArgv": "trim -- tank", "WipefsArgv": "--no-act /dev/a", "ZdbLabelArgv": "-l /dev/a", + "ZdbConfigArgv": "-C tank", "ZgenhostidArgv": "-f 0a0b0c0d", "UdevArgv": "monitor --udev --subsystem-match=block", + } + for name, argv := range pinned { + if got := strings.Join(argv, " "); got != forms[name] { + t.Errorf("%s is %q, the pinned form is %q", name, got, forms[name]) + } + for _, a := range argv { + if (a == "-f" && name != "ZgenhostidArgv") || a == "-r" || a == "-R" || a == "-d" { + t.Errorf("%s holds %s", name, a) + } + } + } + // No argument function of a zpool or zfs program runs a verb ADR 0679 does not allow. + never := map[string]bool{"upgrade": true, "rollback": true, "rename": true, "clear": true, "send": true, "receive": true, "create": true, + "import": true, "export": true, "attach": true, "detach": true, "offline": true, "online": true, "replace": true, "labelclear": true, + "split": true, "reguid": true, "promote": true, "clone": true, "destroy-pool": true, "reopen": true, "sync": true, "checkpoint": true} + for fn, pv := range allowed { + if (pv[0] == "zpool" || pv[0] == "zfs") && never[pv[1]] { + t.Errorf("%s runs %s %s", fn, pv[0], pv[1]) + } + if pv[1] == "destroy" && fn != "DestroyArgv" || pv[0] == "zdb" && fn != "ZdbLabelArgv" && fn != "ZdbConfigArgv" || pv[0] == "zgenhostid" && fn != "ZgenhostidArgv" { + t.Errorf("%s runs %s %s", fn, pv[0], pv[1]) + } + } + // zgenhostid only with the value the parse validated; a device only as cache or log; a destroy only of a snapshot. + files, _ := filepath.Glob("*.go") + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + b, _ := os.ReadFile(file) + for i, l := range strings.Split(string(b), "\n") { + trimmed := strings.TrimSpace(l) + if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "func ") { + continue + } + if strings.Contains(l, "ZgenhostidArgv(") && !strings.Contains(l, "ZgenhostidArgv(c.HostID)") { + t.Errorf("%s:%d runs zgenhostid with a value the parse did not validate: %s", file, i+1, trimmed) + } + if m := regexp.MustCompile(`AddArgv\(\w+(?:\.\w+)?, ("[a-z]+"|\w+),`).FindStringSubmatch(l); m != nil && m[1] != `"cache"` && m[1] != `"log"` { + t.Errorf("%s:%d adds a device that is neither cache nor log: %s", file, i+1, trimmed) + } + } + } for _, act := range []string{"start", "pause", "stop", "anything"} { argv := ScrubArgv(act, "tank") if argv[0] != "scrub" || argv[len(argv)-2] != "--" || argv[len(argv)-1] != "tank" || len(argv) > 4 { @@ -360,7 +426,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "follow.go: if err := cmd.Start(); err != nil {": true, "follow_wake.go: if err := cmd.Start(); err != nil {": true, } - files, _ := filepath.Glob("*.go") + files, _ = filepath.Glob("*.go") sites := 0 for _, file := range files { if strings.HasSuffix(file, "_test.go") { From c6844fc95189156619eb2f2ad03de8b41996126c Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:03:17 +0100 Subject: [PATCH 17/35] Z11 Z12 Z15 the ledger is one instance per dataset and kind, its state a map taken keyed by the ZFS snapshot's name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §21, §22, §26, §28, A-36 (corrects A-30) --- modules/zfs/cmd/zfs-tools/bus.go | 54 +++++++++++-------- modules/zfs/cmd/zfs-tools/health.go | 2 +- modules/zfs/cmd/zfs-tools/health_module.go | 5 +- modules/zfs/cmd/zfs-tools/snapshots_take.go | 18 ++++--- modules/zfs/cmd/zfs-tools/snapshots_test.go | 10 ++-- modules/zfs/cmd/zfs-tools/tools.go | 2 +- modules/zfs/cmd/zfs-tools/tools_config.go | 27 ++++++++-- .../zfs/cmd/zfs-tools/tools_config_test.go | 13 ++--- 8 files changed, 85 insertions(+), 46 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 2760a8e64..8a8f4ba8f 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -136,30 +136,42 @@ func (busLedger) Apply(instance, operation string, body any, call string) error return err } -func (busLedger) Entries() ([]LedgerEntry, error) { - raw, err := stdio.BusState(buscalls.LedgerQuery, buscalls.QueryRequest{Ledger: LedgerName, Index: "by-dataset-kind"}) - if err != nil { - return nil, err - } - var q buscalls.QueryAnswer - if err := json.Unmarshal(raw, &q); err != nil { - return nil, err +// absentInstance says whether a ledger get said the instance is not there, which an instance never written is. +func absentInstance(err error) bool { + s := strings.ToLower(err.Error()) + for _, w := range []string{"not found", "no instance", "no such", "does not exist", "unknown instance", "not kept"} { + if strings.Contains(s, w) { + return true + } } + return false +} + +func (busLedger) Entries(datasets []string) ([]LedgerEntry, error) { var out []LedgerEntry - for _, ie := range q.Entries { - raw, err := stdio.BusState(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: LedgerName, Instance: ie.Instance}) - if err != nil { - return nil, err + for _, ds := range datasets { + for _, kind := range keepKinds { + raw, err := stdio.BusState(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: LedgerName, Instance: LedgerInstance(kind, ds)}) + if err != nil { + if absentInstance(err) { + continue + } + return nil, err + } + var st buscalls.InstanceState + if err := json.Unmarshal(raw, &st); err != nil { + return nil, err + } + var body struct { + Taken map[string]LedgerEntry `json:"taken"` + } + if err := json.Unmarshal(st.State, &body); err != nil { + return nil, err + } + for _, name := range sortedKeys(body.Taken) { + out = append(out, body.Taken[name]) + } } - var st buscalls.InstanceState - if err := json.Unmarshal(raw, &st); err != nil { - return nil, err - } - var e LedgerEntry - if err := json.Unmarshal(st.State, &e); err != nil { - return nil, err - } - out = append(out, e) } return out, nil } diff --git a/modules/zfs/cmd/zfs-tools/health.go b/modules/zfs/cmd/zfs-tools/health.go index 7bfad37f7..7aa45b1d9 100644 --- a/modules/zfs/cmd/zfs-tools/health.go +++ b/modules/zfs/cmd/zfs-tools/health.go @@ -211,7 +211,7 @@ func (m *Machine) Health(ctx context.Context) Health { } sched, serr := m.ScheduleNow(ctx) return m.Errors.Judge(Look{Pools: pools, Status: status, Now: m.Now(), Schedule: sched, ScheduleErr: serr, - Served: m.readServed(ServedFile), Module: m.moduleLook()}) + Served: m.readServed(ServedFile), Module: m.moduleLook(ctx)}) } func sentence(s string) string { diff --git a/modules/zfs/cmd/zfs-tools/health_module.go b/modules/zfs/cmd/zfs-tools/health_module.go index e975e32e9..b58dc0b5f 100644 --- a/modules/zfs/cmd/zfs-tools/health_module.go +++ b/modules/zfs/cmd/zfs-tools/health_module.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "strings" @@ -34,7 +35,7 @@ var snapshotLate = map[string]time.Duration{"hourly": 2 * time.Hour, "daily": 36 // moduleLook reads what the module itself keeps. It answers nil when the config file is not readable or not valid: // the module is then judged on its pools alone, and the refusal is already in the state `applied`. -func (m *Machine) moduleLook() *ModuleLook { +func (m *Machine) moduleLook(ctx context.Context) *ModuleLook { text, err := m.ReadFile(ConfigFile) if err != nil { return nil @@ -58,7 +59,7 @@ func (m *Machine) moduleLook() *ModuleLook { } l.Paused = m.Paused() if m.Ledger != nil { - if es, err := m.Ledger.Entries(); err == nil { + if es, err := m.ledgerAll(ctx); err == nil { l.Ledger, l.LedgerKnown = es, true } } diff --git a/modules/zfs/cmd/zfs-tools/snapshots_take.go b/modules/zfs/cmd/zfs-tools/snapshots_take.go index 977618db5..ab5cd49b2 100644 --- a/modules/zfs/cmd/zfs-tools/snapshots_take.go +++ b/modules/zfs/cmd/zfs-tools/snapshots_take.go @@ -47,10 +47,10 @@ type LedgerEntry struct { At string `json:"at"` } -// LedgerInstance is the instance path of a scheduled snapshot: `snapshot/--`, the -// dataset's `/` written `~`. -func LedgerInstance(kind, stamp, dataset string) string { - return "snapshot/" + kind + "-" + stamp + "-" + strings.ReplaceAll(dataset, "/", "~") +// LedgerInstance is the instance path of a dataset's scheduled snapshots of one kind: `snapshot/-`, the +// dataset's `/` written `~` (A-36). Its state is the map `taken`, keyed by the ZFS snapshot's name. +func LedgerInstance(kind, dataset string) string { + return "snapshot/" + kind + "-" + strings.ReplaceAll(dataset, "/", "~") } // InstanceOf is the instance of a record's snapshot, refusing a name that is not a scheduled snapshot's. @@ -62,7 +62,7 @@ func InstanceOf(dataset, name string) (string, error) { if dataset == "" || strings.ContainsAny(dataset, "@~ ") { return "", fmt.Errorf("%q is not a dataset's name", dataset) } - return LedgerInstance(m[1], m[2], dataset), nil + return LedgerInstance(m[1], dataset), nil } // ledgerEntries reads the ledger through the module's tool zfs_snapshots_schedule. @@ -252,8 +252,8 @@ const LedgerName = "snapshots" type LedgerStore interface { // Apply applies an operation to an instance; call is the id of the tool call it serves. Apply(instance, operation string, body any, call string) error - // Entries are every instance's state. - Entries() ([]LedgerEntry, error) + // Entries are the entries of the `taken` map of every kind's instance of each dataset; a dataset with none has none. + Entries(datasets []string) ([]LedgerEntry, error) } // Record applies the run's `taken` or `removed` to the ledger, refusing a snapshot not named mesh- (A-20). @@ -269,7 +269,9 @@ func (m *Machine) Record(op string, e LedgerEntry, call string) error { return fmt.Errorf("this process reaches no ledger") } if op == "removed" { - return m.Ledger.Apply(instance, op, struct{}{}, call) + return m.Ledger.Apply(instance, op, struct { + Name string `json:"name"` + }{e.Name}, call) } return m.Ledger.Apply(instance, op, e, call) } diff --git a/modules/zfs/cmd/zfs-tools/snapshots_test.go b/modules/zfs/cmd/zfs-tools/snapshots_test.go index b2aec5650..4ec20f2e6 100644 --- a/modules/zfs/cmd/zfs-tools/snapshots_test.go +++ b/modules/zfs/cmd/zfs-tools/snapshots_test.go @@ -131,23 +131,25 @@ func TestADueKindTakesOneSnapshotPerFilesystemAndRecordsEach(t *testing.T) { } // A-20: the tool zfs_snapshots_record refuses a snapshot not named mesh- and one of another operation. -type fakeLedger struct{ applied []string } +type fakeLedger struct{ applied, bodies []string } func (l *fakeLedger) Apply(instance, op string, body any, call string) error { l.applied = append(l.applied, op+" "+instance) + b, _ := json.Marshal(body) + l.bodies = append(l.bodies, string(b)) return nil } -func (l *fakeLedger) Entries() ([]LedgerEntry, error) { return nil, nil } +func (l *fakeLedger) Entries([]string) ([]LedgerEntry, error) { return nil, nil } func TestTheLedgerTakesOnlyTheRunsSnapshots(t *testing.T) { l := &fakeLedger{} m := newFake(t).machine() m.Ledger = l e := LedgerEntry{Dataset: "tank/media", Name: "mesh-hourly-20261009T1000Z", Kind: "hourly", At: "2026-10-09T10:00:00Z"} - if err := m.Record("taken", e, "id"); err != nil || len(l.applied) != 1 || l.applied[0] != "taken snapshot/hourly-20261009T1000Z-tank~media" { + if err := m.Record("taken", e, "id"); err != nil || len(l.applied) != 1 || l.applied[0] != "taken snapshot/hourly-tank~media" { t.Fatalf("%v %v", err, l.applied) } - if err := m.Record("removed", e, "id"); err != nil || l.applied[1] != "removed snapshot/hourly-20261009T1000Z-tank~media" { + if err := m.Record("removed", e, "id"); err != nil || l.applied[1] != "removed snapshot/hourly-tank~media" || l.bodies[1] != `{"name":"mesh-hourly-20261009T1000Z"}` || !strings.Contains(l.bodies[0], `"name":"mesh-hourly-20261009T1000Z"`) { t.Fatal(err, l.applied) } for _, bad := range []LedgerEntry{{Dataset: "tank", Name: "before-move"}, {Dataset: "tank", Name: "mesh-yearly-20261009T1000Z"}, diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go index 8fee83e58..6a0adda26 100644 --- a/modules/zfs/cmd/zfs-tools/tools.go +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -114,7 +114,7 @@ func Tools(m *Machine) []stdio.Tool { readTool("zfs_snapshots_schedule", "The schedule of the scheduled ZFS snapshots: the keep counts, when each kind next falls due, whether it is paused, "+ "the entries past their keep count the next run would remove, and the ledger `{\"ledger\": [...]}` of every scheduled ZFS snapshot the run took "+ "and did not remove. The scheduled run reads the ledger from here. Changes nothing. (r)", nil, - func(map[string]any) (any, error) { return m.SnapshotSchedule() }), + func(map[string]any) (any, error) { return m.SnapshotSchedule(ctx) }), readTool("zfs_module_events", "The module's own events, newest first, from its events stream (config.applied, config.refused, config.waiting, "+ "snapshot.taken.person and .schedule, snapshot.removed.person and .schedule, snapshot.skipped.person and .schedule, run.skipped, scrub.requested), "+ "the last N of one kind or of all, with how many of each kind. Not ZFS's own events: zfs_events reads those. Changes nothing. (r)", diff --git a/modules/zfs/cmd/zfs-tools/tools_config.go b/modules/zfs/cmd/zfs-tools/tools_config.go index 156b4ce39..875f59188 100644 --- a/modules/zfs/cmd/zfs-tools/tools_config.go +++ b/modules/zfs/cmd/zfs-tools/tools_config.go @@ -256,12 +256,12 @@ func wouldRemove(entries []LedgerEntry, k *Keeps) []LedgerEntry { // SnapshotSchedule shows the schedule of the scheduled ZFS snapshots, its next run, what the next run would remove and // the ledger. The ledger is shown even when the config says nothing of it: the scheduled run reads it from here. -func (m *Machine) SnapshotSchedule() (ScheduleAnswer, error) { +func (m *Machine) SnapshotSchedule(ctx context.Context) (ScheduleAnswer, error) { a := ScheduleAnswer{Keeps: map[string]int{}, Next: map[string]string{}, WouldRemove: []LedgerEntry{}, Ledger: []LedgerEntry{}, Said: []string{}} if m.Ledger == nil { return a, fmt.Errorf("this process reaches no ledger") } - entries, err := m.Ledger.Entries() + entries, err := m.ledgerAll(ctx) if err != nil { return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) } @@ -379,7 +379,7 @@ func (m *Machine) CheckSnapshots(ctx context.Context) (CheckAnswer, error) { if m.Ledger == nil { return a, fmt.Errorf("this process reaches no ledger") } - entries, err := m.Ledger.Entries() + entries, err := m.ledgerAll(ctx) if err != nil { return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) } @@ -445,3 +445,24 @@ func (m *Machine) FindingsNow() (FindingsAnswer, error) { } return a, nil } + +// ledgerAll reads the ledger's entries for every filesystem of the config's pool (A-36: the ledger is one instance per +// dataset and kind, so the datasets name what to read). No pool set, no entries. +func (m *Machine) ledgerAll(ctx context.Context) ([]LedgerEntry, error) { + text, err := m.ReadFile(ConfigFile) + if err != nil { + return nil, fmt.Errorf("%s could not be read: %v", ConfigFile, err) + } + var c struct { + Pool string `json:"pool"` + } + _ = json.Unmarshal(text, &c) + if c.Pool == "" { + return nil, nil + } + fs, _, err := m.filesystems(ctx, c.Pool) + if err != nil { + return nil, err + } + return m.Ledger.Entries(fs) +} diff --git a/modules/zfs/cmd/zfs-tools/tools_config_test.go b/modules/zfs/cmd/zfs-tools/tools_config_test.go index 5e55fca27..edb15491d 100644 --- a/modules/zfs/cmd/zfs-tools/tools_config_test.go +++ b/modules/zfs/cmd/zfs-tools/tools_config_test.go @@ -16,7 +16,7 @@ type staticLedger struct { } func (l staticLedger) Apply(string, string, any, string) error { return nil } -func (l staticLedger) Entries() ([]LedgerEntry, error) { return l.entries, l.err } +func (l staticLedger) Entries([]string) ([]LedgerEntry, error) { return l.entries, l.err } type staticLog struct { evs []LoggedEvent @@ -124,11 +124,11 @@ func TestSnapshotsScheduleAnswersTheLedgerTheRunReads(t *testing.T) { {name: "paused", conf: `{"pool":"tank","snapshots":{"hourly":1}}`, paused: true, said: "paused"}, {name: "none keeps what exists", conf: `{"pool":"tank","snapshots":{"none":true}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "are kept"}, {name: "not managed", conf: `{"pool":"tank"}`, said: "not managed"}, - {name: "refused config still answers the ledger", conf: `{"snapshots":{"hourly":999}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "refused"}, - {name: "ledger unreadable", conf: `{}`, l: staticLedger{err: errors.New("no bus")}, wantErr: true}, + {name: "refused config still answers the ledger", conf: `{"pool":"tank","snapshots":{"hourly":999}}`, l: staticLedger{entries: []LedgerEntry{e(1)}}, said: "refused"}, + {name: "ledger unreadable", conf: `{"pool":"tank"}`, l: staticLedger{err: errors.New("no bus")}, wantErr: true}, } for _, c := range cases { - f := newFake(t) + f := withZFS(t) f.now = time.Date(2026, 10, 9, 12, 30, 0, 0, time.UTC) // a Friday m := f.machine() st := memStates{} @@ -137,7 +137,7 @@ func TestSnapshotsScheduleAnswersTheLedgerTheRunReads(t *testing.T) { if c.paused { st.Put(StateSchedule, KeyPaused, map[string]string{"by": "x", "why": "y"}) } - a, err := m.SnapshotSchedule() + a, err := m.SnapshotSchedule(context.Background()) if (err != nil) != c.wantErr { t.Errorf("%s: %v", c.name, err) continue @@ -195,9 +195,10 @@ func TestModuleEventsAreTheStreamsLastNNewestFirstByKind(t *testing.T) { // §28: zfs_snapshots_check names each ledger entry whose ZFS snapshot is gone and each mesh- snapshot the ledger // does not hold, and changes nothing. func TestSnapshotsCheckNamesWhatTheLedgerAndZFSDoNotShare(t *testing.T) { - f := newFake(t) + f := withZFS(t) f.on(line("zfs", SnapshotsArgv()), "tank@mesh-hourly-20261009T1000Z\t1791400000\t1\t2\ntank@mesh-hourly-20261009T1100Z\t1791400000\t1\t2\ntank@by-hand\t1791400000\t1\t2\n") f.uid = 0 + f.files[ConfigFile] = `{"pool":"tank"}` m := f.machine() m.Ledger = staticLedger{entries: []LedgerEntry{{"tank", "mesh-hourly-20261009T1000Z", "hourly", "2026-10-09T10:00:00Z"}, {"tank", "mesh-hourly-20261009T0900Z", "hourly", "2026-10-09T09:00:00Z"}}} From ec27f12524011d1c1c9fd5e4c9841a48ebe67b8a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:04:41 +0100 Subject: [PATCH 18/35] A-37 the keeper: the module's one process bus account, a socket only root opens, and the root processes' bus through it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §18, §20, §26, §27, A-6, A-20, A-37 --- modules/zfs/cmd/zfs-tools/bus.go | 50 +++- modules/zfs/cmd/zfs-tools/keeper.go | 312 +++++++++++++++++++++++ modules/zfs/cmd/zfs-tools/keeper_test.go | 124 +++++++++ modules/zfs/cmd/zfs-tools/main.go | 5 + modules/zfs/go.mod | 2 +- 5 files changed, 480 insertions(+), 13 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/keeper.go create mode 100644 modules/zfs/cmd/zfs-tools/keeper_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 8a8f4ba8f..347e0d57e 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -1,7 +1,9 @@ package main import ( + "context" "encoding/json" + "fmt" "os" "strings" "sync" @@ -33,25 +35,49 @@ func (b busStates) Get(state, key string) (json.RawMessage, bool, error) { return e.Value, true, nil } -// JoinBus gives the machine this process's events and states, as its own bus account says. Without the -// account or the bus the machine keeps none, and what it changes is said as a warning (§18: with the bus down -// the apply converges from the config file it last got). It answers how to leave the bus. +// JoinBus gives a root process the module's events and states through the keeper's socket (A-37): the process holds no +// bus account of its own. Without the socket the machine keeps none, and what it changes is said as a warning (§18: +// with the bus down the apply converges from the config file it last got). It answers how to leave the bus. func (m *Machine) JoinBus() func() { + sock := strings.TrimSpace(os.Getenv(KeeperSocketEnv)) + if sock == "" { + m.Warnings = append(m.Warnings, KeeperSocketEnv+" is not set, so the keeper is not reached and the bus is taken as down") + return func() {} + } + c := keeperClient{sock} + // The keeper answering is what the bus being up means: one call says it before the process relies on it. + if _, err := c.Keys(StateSchedule); err != nil { + m.Warnings = append(m.Warnings, err.Error()+": the bus is taken as down") + return func() {} + } + m.Emit = c.emit + m.States = c + m.Findings = meshconditionsclient.FindingWriter{State: findingState{c}} + m.Call = c.call + return func() {} +} + +// RunKeeper is the process `zfs-keeper`: the module's one process bus account, serving the socket to root. +func RunKeeper(ctx context.Context) error { + sock := strings.TrimSpace(os.Getenv(KeeperSocketEnv)) + if sock == "" { + return fmt.Errorf("%s is not set", KeeperSocketEnv) + } acct, err := process.ReadBrokerAccount(os.Getenv) if err != nil { - m.Warnings = append(m.Warnings, err.Error()) - return func() {} + return err } c, err := process.Connect(acct) if err != nil { - m.Warnings = append(m.Warnings, err.Error()) - return func() {} + return err } - m.Emit = func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) } - m.States = busStates{c} - m.Findings = meshconditionsclient.FindingWriter{State: c.State(StateFindings)} - m.Call = c.Call - return c.Close + defer c.Close() + l, err := Listen(sock) + if err != nil { + return err + } + k := &Keeper{Emit: func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) }, States: busStates{c}, Call: c.Call} + return k.Serve(ctx, l) } // toolStates is the module's states as the tools bundle reaches them, through the tool runner. diff --git a/modules/zfs/cmd/zfs-tools/keeper.go b/modules/zfs/cmd/zfs-tools/keeper.go new file mode 100644 index 000000000..fc63abf7e --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/keeper.go @@ -0,0 +1,312 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +// The keeper (design 42 zfs-owns-zfs A-37, §18, §20, §26, §27; ADR 0436 §1, §2). +// +// A process's bus account is a user the module makes, never root, so the three root processes (the apply, the run and +// the event follower) declare no `bus`. The process `zfs-keeper`, running as the module's user `zfs-keeper`, holds the +// module's one process bus account and serves a unix socket in the module's state directory that only root may open +// (the peer's uid is checked on every connection). Its calls are a closed list: emit an event of the module, put or +// delete a key of its states `applied`, `schedule` and `findings`, and invoke one of three (`zfs.zfs_snapshots_record`, +// `zfs.zfs_snapshots_schedule` and `seat:node-service-manager.start`, of the two units of the module only). The root +// processes say everything they say on the bus through it; with the keeper not answering they act as with the bus down +// (A-8), and warn. +// +// Reading a key or the keys of those states (`get`, `keys`) is also served: the run reads the hand acts waiting in the +// state `schedule` and the apply reads the state `applied` (A-17), and a root process has no other way to the bus. + +// KeeperSocketEnv names the variable holding the socket's path, in every process of the module that uses it. +const KeeperSocketEnv = "ZFS_KEEPER_SOCKET" + +// KeeperTimeout bounds one call through the keeper. +const KeeperTimeout = 30 * time.Second + +// The calls of the keeper. +const ( + OpEmit = "emit" + OpPut = "put" + OpDelete = "delete" + OpGet = "get" + OpKeys = "keys" + OpInvoke = "invoke" +) + +// KeeperRequest is one call: one JSON line. +type KeeperRequest struct { + Op string `json:"op"` + Kind string `json:"kind,omitempty"` + State string `json:"state,omitempty"` + Key string `json:"key,omitempty"` + Body json.RawMessage `json:"body,omitempty"` +} + +// KeeperAnswer is its answer: one JSON line. +type KeeperAnswer struct { + Error string `json:"error,omitempty"` + Found bool `json:"found,omitempty"` + Value json.RawMessage `json:"value,omitempty"` + Keys []string `json:"keys,omitempty"` +} + +var keeperStates = map[string]bool{StateApplied: true, StateSchedule: true, StateFindings: true} + +// Keeper serves the calls over the bus account of its own process. +type Keeper struct { + Emit func(kind string, body any) error + States StateStore + Call func(key string, body any) (json.RawMessage, error) + // PeerOf is the uid at the other end of a connection; nil reads it from the socket (SO_PEERCRED). + PeerOf func(net.Conn) (int, error) +} + +// invokable says whether a call key is one of the three, and for the service manager whether it starts one of the +// module's two units. +func invokable(key string, body json.RawMessage) error { + switch { + case key == ToolRecord || key == ToolSchedule: + return nil + case strings.HasPrefix(key, "seat:node-service-manager.start@"): + var a map[string]any + if json.Unmarshal(body, &a) != nil || a["scope"] != "system" || (a["unit"] != RunService && a["unit"] != ApplyUnit) || len(a) != 2 { + return fmt.Errorf("the keeper starts only %s or %s, in the system scope", RunService, ApplyUnit) + } + return nil + } + return fmt.Errorf("%q is not one of the three calls the keeper invokes", key) +} + +// Do answers one request. +func (k *Keeper) Do(r KeeperRequest) KeeperAnswer { + fail := func(err error) KeeperAnswer { return KeeperAnswer{Error: err.Error()} } + switch r.Op { + case OpEmit: + known := false + for _, e := range EventKinds { + known = known || (e == r.Kind && e != EventScrubRequested) + } + if !known { + return fail(fmt.Errorf("%q is not an event the module's processes say", r.Kind)) + } + var body any + if len(r.Body) > 0 { + if err := json.Unmarshal(r.Body, &body); err != nil { + return fail(err) + } + } + if err := k.Emit(r.Kind, body); err != nil { + return fail(err) + } + return KeeperAnswer{} + case OpPut, OpDelete, OpGet, OpKeys: + if !keeperStates[r.State] { + return fail(fmt.Errorf("%q is not a state of the module's processes (applied, schedule, findings)", r.State)) + } + switch r.Op { + case OpPut: + var v any + if err := json.Unmarshal(r.Body, &v); err != nil { + return fail(err) + } + if err := k.States.Put(r.State, r.Key, v); err != nil { + return fail(err) + } + case OpDelete: + if err := k.States.Delete(r.State, r.Key); err != nil { + return fail(err) + } + case OpGet: + v, ok, err := k.States.Get(r.State, r.Key) + if err != nil { + return fail(err) + } + return KeeperAnswer{Found: ok, Value: v} + case OpKeys: + keys, err := k.States.Keys(r.State) + if err != nil { + return fail(err) + } + return KeeperAnswer{Keys: keys} + } + return KeeperAnswer{} + case OpInvoke: + if err := invokable(r.Key, r.Body); err != nil { + return fail(err) + } + var body any + _ = json.Unmarshal(r.Body, &body) + out, err := k.Call(r.Key, body) + if err != nil { + return fail(err) + } + return KeeperAnswer{Value: out} + } + return fail(fmt.Errorf("%q is not a call of the keeper (emit, put, delete, get, keys, invoke)", r.Op)) +} + +// peerUID is the account at the other end of a unix socket. +func peerUID(c net.Conn) (int, error) { + uc, ok := c.(*net.UnixConn) + if !ok { + return -1, errors.New("not a unix socket") + } + raw, err := uc.SyscallConn() + if err != nil { + return -1, err + } + uid := -1 + var gerr error + if err := raw.Control(func(fd uintptr) { + cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) + if err != nil { + gerr = err + return + } + uid = int(cred.Uid) + }); err != nil { + return -1, err + } + return uid, gerr +} + +// Serve answers each connection on l, one request each, until ctx ends: a peer that is not root is told so and +// nothing it sent is read. +func (k *Keeper) Serve(ctx context.Context, l net.Listener) error { + go func() { <-ctx.Done(); l.Close() }() + peerOf := k.PeerOf + if peerOf == nil { + peerOf = peerUID + } + for { + c, err := l.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + go func() { + defer c.Close() + _ = c.SetDeadline(time.Now().Add(KeeperTimeout)) + enc := json.NewEncoder(c) + if uid, err := peerOf(c); err != nil || uid != 0 { + _ = enc.Encode(KeeperAnswer{Error: "only root opens the keeper's socket"}) + return + } + line, err := bufio.NewReader(c).ReadBytes('\n') + if err != nil { + return + } + var r KeeperRequest + if err := json.Unmarshal(line, &r); err != nil { + _ = enc.Encode(KeeperAnswer{Error: "the request is not JSON: " + err.Error()}) + return + } + _ = enc.Encode(k.Do(r)) + }() + } +} + +// Listen opens the socket: in a folder of the keeper's own, as the keeper's account, mode 0600 so that only the +// account itself and root open it; any older socket is replaced. +func Listen(sock string) (net.Listener, error) { + if err := os.MkdirAll(filepath.Dir(sock), 0o711); err != nil { + return nil, err + } + _ = os.Remove(sock) + l, err := net.Listen("unix", sock) + if err != nil { + return nil, err + } + if err := os.Chmod(sock, 0o600); err != nil { + l.Close() + return nil, err + } + return l, nil +} + +// keeperClient is a root process's way to the bus: calls on the keeper's socket. +type keeperClient struct{ sock string } + +func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) { + conn, err := net.DialTimeout("unix", c.sock, 5*time.Second) + if err != nil { + return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(KeeperTimeout)) + if err := json.NewEncoder(conn).Encode(r); err != nil { + return KeeperAnswer{}, err + } + var a KeeperAnswer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err) + } + if a.Error != "" { + return a, errors.New(a.Error) + } + return a, nil +} + +func (c keeperClient) emit(kind string, body any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + _, err = c.do(KeeperRequest{Op: OpEmit, Kind: kind, Body: raw}) + return err +} + +func (c keeperClient) Put(state, key string, v any) error { + raw, err := json.Marshal(v) + if err != nil { + return err + } + _, err = c.do(KeeperRequest{Op: OpPut, State: state, Key: key, Body: raw}) + return err +} + +func (c keeperClient) Delete(state, key string) error { + _, err := c.do(KeeperRequest{Op: OpDelete, State: state, Key: key}) + return err +} + +func (c keeperClient) Keys(state string) ([]string, error) { + a, err := c.do(KeeperRequest{Op: OpKeys, State: state}) + return a.Keys, err +} + +func (c keeperClient) Get(state, key string) (json.RawMessage, bool, error) { + a, err := c.do(KeeperRequest{Op: OpGet, State: state, Key: key}) + return a.Value, a.Found, err +} + +func (c keeperClient) call(key string, body any) (json.RawMessage, error) { + raw, err := json.Marshal(body) + if err != nil { + return nil, err + } + a, err := c.do(KeeperRequest{Op: OpInvoke, Key: key, Body: raw}) + return a.Value, err +} + +// findingState is the state `findings` as the finding writer reaches it, through the keeper. +type findingState struct{ c keeperClient } + +func (f findingState) Put(key string, value any) (uint64, error) { + return 0, f.c.Put(StateFindings, key, value) +} +func (f findingState) Delete(key string) error { return f.c.Delete(StateFindings, key) } diff --git a/modules/zfs/cmd/zfs-tools/keeper_test.go b/modules/zfs/cmd/zfs-tools/keeper_test.go new file mode 100644 index 000000000..9c5cbb2cd --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/keeper_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" +) + +// A-37: the keeper answers a closed list of calls: an event of the module (never scrub.requested), a key of its three +// states, and one of three invokes (the service manager only for the module's two units); anything else is refused. +func TestTheKeeperAnswersOnlyItsClosedListOfCalls(t *testing.T) { + var emitted []string + var called []string + st := memStates{} + k := &Keeper{ + Emit: func(kind string, body any) error { emitted = append(emitted, kind); return nil }, + States: st, + Call: func(key string, body any) (json.RawMessage, error) { + called = append(called, key) + return json.RawMessage(`{"ok":true}`), nil + }, + } + req := func(op, kind, state, key, body string) KeeperRequest { + return KeeperRequest{Op: op, Kind: kind, State: state, Key: key, Body: json.RawMessage(body)} + } + cases := []struct { + name string + r KeeperRequest + refused string + }{ + {"emit a kind", req(OpEmit, EventApplied, "", "", `{"key":"x"}`), ""}, + {"emit the scrub's kind", req(OpEmit, EventScrubRequested, "", "", `{}`), "not an event"}, + {"emit another module's kind", req(OpEmit, "package.changed", "", "", `{}`), "not an event"}, + {"put applied", req(OpPut, "", StateApplied, "hostid", `{"outcome":"applied"}`), ""}, + {"put schedule", req(OpPut, "", StateSchedule, "paused", `{}`), ""}, + {"put another state", req(OpPut, "", "holds", "x", `{}`), "not a state"}, + {"get findings", req(OpGet, "", StateFindings, "k", ``), ""}, + {"keys of another state", req(OpKeys, "", "mirrors", "", ``), "not a state"}, + {"delete schedule", req(OpDelete, "", StateSchedule, "paused", ``), ""}, + {"invoke record", req(OpInvoke, "", "", ToolRecord, `{}`), ""}, + {"invoke schedule", req(OpInvoke, "", "", ToolSchedule, `{}`), ""}, + {"start the run", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"zfs-mesh-run.service"}`), ""}, + {"start the apply", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"zfs-config-apply.service"}`), ""}, + {"start another unit", req(OpInvoke, "", "", "seat:node-service-manager.start@ace", `{"scope":"system","unit":"sshd.service"}`), "starts only"}, + {"stop a unit", req(OpInvoke, "", "", "seat:node-service-manager.stop@ace", `{"scope":"system","unit":"zfs-mesh-run.service"}`), "not one of the three"}, + {"invoke a tool of the module", req(OpInvoke, "", "", "zfs.zfs_snapshot", `{}`), "not one of the three"}, + {"another op", req("shell", "", "", "", ``), "not a call"}, + } + for _, c := range cases { + a := k.Do(c.r) + if (c.refused == "") != (a.Error == "") || (c.refused != "" && !strings.Contains(a.Error, c.refused)) { + t.Errorf("%s: %+v", c.name, a) + } + } + if len(emitted) != 1 || len(called) != 4 { + t.Errorf("emitted %v, called %v", emitted, called) + } + if _, left := st["applied/hostid"]; !left { + t.Errorf("the put was not kept") + } +} + +// A-37: only root opens the socket; a root process says and reads through it as through a bus of its own, and with no +// keeper answering it acts as with the bus down, with a warning (A-8). +func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing.T) { + st := memStates{} + uid := 0 + k := &Keeper{Emit: func(string, any) error { return nil }, States: st, + Call: func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }, + PeerOf: func(net.Conn) (int, error) { return uid, nil }} + dir, err := os.MkdirTemp("/var/tmp", "zk") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + sock := filepath.Join(dir, "k", "k.sock") + l, err := Listen(sock) + if err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go k.Serve(ctx, l) + c := keeperClient{sock} + if err := c.Put(StateSchedule, "snapshots", map[string]string{"by": "x"}); err != nil { + t.Fatal(err) + } + if keys, err := c.Keys(StateSchedule); err != nil || len(keys) != 1 || keys[0] != "snapshots" { + t.Errorf("%v %v", keys, err) + } + if v, ok, err := c.Get(StateSchedule, "snapshots"); err != nil || !ok || !strings.Contains(string(v), `"by":"x"`) { + t.Errorf("%s %v %v", v, ok, err) + } + if _, err := c.call(ToolSchedule, map[string]any{}); err == nil || !strings.Contains(err.Error(), "down") { + t.Errorf("an invoke that failed: %v", err) + } + uid = 1000 + if err := c.Put(StateSchedule, "x", 1); err == nil || !strings.Contains(err.Error(), "only root") { + t.Errorf("a peer that is not root: %v", err) + } + if _, left := st["schedule/x"]; left { + t.Errorf("a refused peer's call was read") + } + // No keeper: the machine keeps no bus and says so. + t.Setenv(KeeperSocketEnv, filepath.Join(dir, "none.sock")) + m := newFake(t).machine() + m.Emit = nil + m.JoinBus() + if m.Emit != nil || m.States != nil || len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "bus is taken as down") { + t.Errorf("no keeper: %+v", m.Warnings) + } + t.Setenv(KeeperSocketEnv, sock) + uid = 0 + m2 := newFake(t).machine() + m2.JoinBus() + if m2.Emit == nil || m2.States == nil || m2.Call == nil || m2.Findings == nil { + t.Errorf("a keeper answering: the bus is up") + } +} diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index eff072e8f..b10224215 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -53,6 +53,11 @@ func main() { os.Exit(1) } fmt.Println(said) + case "keeper": + if err := RunKeeper(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } case "events": if err := Follower(ctx, os.Stdout); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/modules/zfs/go.mod b/modules/zfs/go.mod index 804a627cf..8b1859664 100644 --- a/modules/zfs/go.mod +++ b/modules/zfs/go.mod @@ -15,5 +15,5 @@ require ( github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/sys v0.48.0 // indirect + golang.org/x/sys v0.48.0 ) From 8927d689003bda13c2327f980e38fe87b38529de Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:05:19 +0100 Subject: [PATCH 19/35] Z4 zfs-manifest-bus: the events stream, states and ledger by template, the keeper as the module's one process bus account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three root processes declare no bus and reach the keeper's socket; the module invokes the service manager's start. The judge refuses the keeper's bus emits and state until the validator accepts what the module asks for in requires (A-38). Builds: design 42 zfs-owns-zfs §25, §26, §27, §28, A-6, A-19, A-36, A-37, A-38 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 85 ++++++++++- modules/zfs/module.json | 169 ++++++++++++++++++++- 2 files changed, 247 insertions(+), 7 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 3c028f2a0..ea7021441 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -76,8 +76,8 @@ func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T if len(m.Claims) != 0 { t.Errorf("the module claims no seat: %v", m.Claims) } - if strings.Join(m.Emits, ",") != "snapshot.taken,"+ScrubEvent { - t.Errorf("emits %v", m.Emits) + if len(m.Emits) != 0 { + t.Errorf("the events are asked for by template in requires: emits %v", m.Emits) } if hit := regexp.MustCompile(`\b(g14|novox|ace|shanks|jochen)\b|/home/`).FindString(strings.ReplaceAll(raw, "novox/hq", "")); hit != "" { t.Errorf("the manifest says %q", hit) @@ -459,3 +459,84 @@ func TestTheBundleRunsOnlyTheFormsOfADR0679(t *testing.T) { t.Errorf("found %d call sites; the test no longer sees them", sites) } } + +// §25, §26 (A-19, A-36), A-37, A-38, §28: one events stream by template, 30 days, every kind of §25; the states applied, +// schedule and findings; the ledger snapshots of A-36 with a fixed cap and no index; the keeper alone holds the process bus, as a user of +// the module's own, naming the module's events and states and invoking the three; the three root processes declare no bus and +// reach the keeper's socket; the module invokes the service manager's start; the tools list holds §28's. +func TestTheManifestsBusStateKeeperAndTools(t *testing.T) { + tree, byID, _ := manifestOf(t) + reqs := map[string]map[string]any{} + for _, r := range tree["requires"].([]any) { + e := r.(map[string]any) + reqs[e["name"].(string)] = e + } + ev := reqs["events"] + opts, _ := ev["options"].(map[string]any) + var kinds []string + for _, k := range opts["events"].([]any) { + kinds = append(kinds, k.(string)) + } + if ev["storage"] != "stream" || ev["template"] != "events" || opts["max-age-days"] != float64(30) || strings.Join(kinds, ",") != strings.Join(EventKinds, ",") { + t.Errorf("the events stream: %v", ev) + } + for _, n := range []string{"applied", "schedule", "findings"} { + if e := reqs[n]; e == nil || e["storage"] != "key-value" || e["template"] != "state" { + t.Errorf("the state %s: %v", n, e) + } + } + if reqs["findings"]["options"] != nil { + t.Errorf("findings declares no stream of its own (A-32): %v", reqs["findings"]) + } + l := reqs[LedgerName] + lo, _ := l["options"].(map[string]any) + types, _ := l["types"].([]any) + if l["storage"] != "ledger" || l["template"] != "ledger" || lo["max-mib"] == nil || len(types) != 1 { + t.Fatalf("the ledger: %v", l) + } + ty := types[0].(map[string]any) + ops, _ := json.Marshal(ty["operations"]) + if ty["name"] != "snapshot" || ty["indexes"] != nil || + string(ops) != `[{"effect":"set-entry","fields":["dataset","name","kind","at"],"into":"taken","key":"name","name":"taken"},{"effect":"remove","into":"taken","key":"name","name":"removed"}]` { + t.Errorf("the ledger's type: %v", ty) + } + k := byID["keeper"] + bus, _ := k["bus"].(map[string]any) + acct := byID["keeper-account"] + if k["user"] != "zfs-keeper" || acct["type"] != "user" || acct["name"] != "zfs-keeper" || bus == nil || + fmt.Sprint(bus["state"]) != "[applied schedule findings]" || + fmt.Sprint(bus["invokes"]) != "["+ToolRecord+" "+ToolSchedule+" seat:node-service-manager.start]" { + t.Errorf("the keeper: %v %v", k, acct) + } + if got := fmt.Sprint(bus["emits"]); strings.Contains(got, ScrubEvent) || !strings.Contains(got, EventApplied) || !strings.Contains(got, EventTakenSchedule) { + t.Errorf("the keeper's events: %v", got) + } + for _, id := range []string{"apply", "events", "run"} { + r := byID[id] + if r["bus"] != nil || fmt.Sprint(r["env"]) != "map["+KeeperSocketEnv+":${dir:keeper-state}/keeper.sock]" { + t.Errorf("%s declares a bus or does not reach the keeper's socket: %v", id, r) + } + } + for _, id := range []string{"watch", "keeper-state"} { + if byID[id]["bus"] != nil { + t.Errorf("%s", id) + } + } + if byID["keeper-state"]["owner"] != "zfs-keeper" { + t.Errorf("the keeper's folder: %v", byID["keeper-state"]) + } + if fmt.Sprint(tree["invokes"]) != "[seat:node-service-manager.start]" { + t.Errorf("invokes: %v", tree["invokes"]) + } + m, _ := readManifest(t) + for _, tool := range []string{"zfs_config", "zfs_config_apply", "zfs_snapshots_schedule", "zfs_snapshots_run_now", "zfs_snapshots_prune_now", "zfs_snapshots_pause", + "zfs_snapshots_resume", "zfs_module_events", "zfs_snapshots_check", "zfs_findings", "zfs_snapshots_record"} { + found := false + for _, x := range m.Tools { + found = found || x == tool + } + if !found { + t.Errorf("%s is not in tools", tool) + } + } +} diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 2f119f5ab..fd2b98256 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -5,9 +5,103 @@ "package-manager", "service-manager" ], - "emits": [ - "snapshot.taken", - "scrub.requested" + "invokes": [ + "seat:node-service-manager.start" + ], + "own-secrets": { + "broker-keeper": { + "path": "${dir:keeper-state}/broker-keeper.secret", + "owner": "zfs-keeper" + } + }, + "requires": [ + { + "name": "events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "config.applied", + "config.refused", + "config.waiting", + "snapshot.taken.person", + "snapshot.taken.schedule", + "snapshot.removed.person", + "snapshot.removed.schedule", + "snapshot.skipped.person", + "snapshot.skipped.schedule", + "run.skipped", + "scrub.requested" + ], + "max-age-days": 30, + "max-mib": 16 + } + }, + { + "name": "applied", + "storage": "key-value", + "template": "state" + }, + { + "name": "schedule", + "storage": "key-value", + "template": "state" + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "snapshots", + "storage": "ledger", + "template": "ledger", + "options": { + "max-mib": 64 + }, + "types": [ + { + "name": "snapshot", + "schema": { + "dataset": { + "type": "string" + }, + "name": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "at": { + "type": "string" + }, + "taken": { + "type": "object" + } + }, + "operations": [ + { + "name": "taken", + "effect": "set-entry", + "into": "taken", + "key": "name", + "fields": [ + "dataset", + "name", + "kind", + "at" + ] + }, + { + "name": "removed", + "effect": "remove", + "into": "taken", + "key": "name" + } + ] + } + ] + } ], "config": { "pool": { @@ -260,6 +354,20 @@ "mode": "0755", "unknown": "program" }, + { + "id": "keeper-account", + "type": "user", + "name": "zfs-keeper", + "shell": "/usr/bin/nologin" + }, + { + "id": "keeper-state", + "type": "directory", + "mode": "0711", + "owner": "zfs-keeper", + "place": "mesh", + "unknown": "program" + }, { "id": "config-dir", "type": "directory", @@ -432,6 +540,48 @@ "zed=off" ] }, + { + "id": "keeper", + "type": "process", + "name": "zfs-keeper", + "artifact": "tools", + "run": [ + "./zfs-tools", + "keeper" + ], + "user": "zfs-keeper", + "env": { + "MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper.secret", + "ZFS_KEEPER_SOCKET": "${dir:keeper-state}/keeper.sock" + }, + "bus": { + "state": [ + "applied", + "schedule", + "findings" + ], + "emits": [ + "config.applied", + "config.refused", + "config.waiting", + "snapshot.taken.person", + "snapshot.taken.schedule", + "snapshot.removed.person", + "snapshot.removed.schedule", + "snapshot.skipped.person", + "snapshot.skipped.schedule", + "run.skipped" + ], + "invokes": [ + "zfs.zfs_snapshots_record", + "zfs.zfs_snapshots_schedule", + "seat:node-service-manager.start" + ] + }, + "health": { + "kind": "unit" + } + }, { "id": "apply", "type": "process", @@ -444,7 +594,10 @@ "run-once": true, "restart-on": [ "config" - ] + ], + "env": { + "ZFS_KEEPER_SOCKET": "${dir:keeper-state}/keeper.sock" + } }, { "id": "watch", @@ -476,6 +629,9 @@ ], "health": { "kind": "unit" + }, + "env": { + "ZFS_KEEPER_SOCKET": "${dir:keeper-state}/keeper.sock" } }, { @@ -487,7 +643,10 @@ "./zfs-tools", "run" ], - "schedule": "0,15,30,45 * * * *" + "schedule": "0,15,30,45 * * * *", + "env": { + "ZFS_KEEPER_SOCKET": "${dir:keeper-state}/keeper.sock" + } }, { "id": "distribution-weekly", From 7831f14a6e8ec1ee089be19b364e4a78fcb0ffb9 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:06:04 +0100 Subject: [PATCH 20/35] Z19 zfs-readme-budget: the README says what the module owns and how it is checked, and the test budget is a test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §31, §7, §19, §23 --- modules/zfs/README.md | 172 +++++++++++++-------- modules/zfs/cmd/zfs-tools/manifest_test.go | 16 ++ 2 files changed, 124 insertions(+), 64 deletions(-) diff --git a/modules/zfs/README.md b/modules/zfs/README.md index ecb0b5dc3..97609eb82 100644 --- a/modules/zfs/README.md +++ b/modules/zfs/README.md @@ -1,65 +1,105 @@ # zfs -The mesh's view of ZFS on a machine: every pool's health, layout, space, scrubs, I/O, cache and events, -and every dataset with its space and snapshots. It owns the pool's scrub schedule, run by a scheduled process of its own. It has four acts: a named -snapshot of one dataset, and a scrub's start, pause and stop. It is for the home server, whose pool holds -the operator's media library, which the mesh does not own (novox/hq ADR 0051). So the module reads, and -what it does only adds a snapshot or reads the pool through. +The mesh's view of ZFS on a machine, and what it owns of it: the packages, the pool's properties, cache and log devices, +the node's ZFS id, the ARC, the event daemon, the scrub and trim schedule and scheduled ZFS snapshots. It changes only what +its config values set: a value that is not set, or an empty list, is not managed, and nothing is ever removed by a blank +value (ADR 0679). The pool's data is the operator's, which the mesh does not own (novox/hq ADR 0051). + +## What it installs + +`zfs-utils` and `zfs-dkms` are declared present: the node-engine installs a missing one, upgrades none, removes none. +`zfs-utils-debug` is not the module's and is left as found. The module never writes the package's own `/etc/default/zfs` +or `/etc/zfs/zed.d`; a `.pacnew` beside `/etc/default/zfs` or the package's `zed.rc` is a warning. ## What it does not do -There is no destroy, rollback, set, send, receive, rename, clone, replace, attach, detach or clear. Each of -those changes or removes the operator's data, or the pool's layout, and needs a decision of its own first. -A resilver is ZFS's own: it starts one when a disk is replaced or comes back. The module watches and -reports resilvers, and starts none. +It never destroys a dataset, rolls back, renames, clears, sends, receives, replaces, attaches or detaches, and it adds +no data vdev, spare, special or dedup device. Each of those changes or removes the operator's data or the pool's layout +and needs a decision of its own first. A dataset's `sharenfs` is the seat `node-nfs-server`'s and `sharesmb` the SMB module's: +a value naming either is refused. -This is checked by `TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub`. Every command the bundle runs is -built by one of its argument functions, and each function is held to its program and verb: +This is checked by `TestTheBundleRunsOnlyTheFormsOfADR0679`. Every command the bundle runs is built by one of its argument +functions, and each function is held to its program, its verb and its whole form: -- `zfs`: `list`, `get` and `snapshot`; -- `zpool`: `list`, `status`, `iostat`, `events`, `history`, `wait -t scrub` and `scrub` (with `-p` or `-s` - only, and those only from the person's acts, never from the scheduled run); -- `systemctl`: `show`. +- `zpool`: `add cache|log` (never `-f`), `remove` of a cache or log device, `set` of a listed pool property, `scrub` + (with `-p` or `-s` only from a person's act), `trim` of the pool, and the reads `status`, `list`, `get`, `iostat`, `events` + and `history`; +- `zfs`: `snapshot`, `set mesh:schedule=` on a snapshot named `mesh-`, `destroy -- @`, `set` and + `inherit` of a listed dataset property, and the reads `list`, `get` and `holds`; +- `wipefs --no-act `, `zdb -l ` and `zdb -C `, `zgenhostid -f ` only with the value the parse + validated, `udevadm monitor --udev --subsystem-match=block`, and `systemctl show`. -The test fails on any call site that builds its own argument list, through `ask`, `AsRoot`, `Escalated`, -`.Run(`, `exec.Command(`, `exec.CommandContext(`, `os.StartProcess` or `syscall.Exec`. +Any other `zpool` or `zfs` form, and every `-f`, `-r`, `-R` and `-d`, is refused; so is any call site that builds its own argument +list. -It installs nothing. ZFS belongs to the machine, and its kernel module is built against that machine's -kernel. On a machine without ZFS, every tool answers `applicable: false` with the reason, and the health -check is unhealthy and says to unassign the module, because a module put where it does nothing must not -look healthy. +On a machine without ZFS, every tool answers `applicable: false` with the reason, and the health check is unhealthy and says +to unassign the module. It holds no seat. A new `node-*` seat is the controller's to define (novox/hq ADR 0121, ADR 0246). +## Config values + +| value | form | managed by | +|---|---|---| +| `pool` (was `scrub-pool`) | the pool's name, trusted, required | the scrub, trim, ZFS snapshots and the apply | +| `scrub-cadence`, `scrub-day`, `scrub-time` | as before, with their defaults | the scheduled run | +| `arc-max`, `arc-min` | whole MiB | written live and into `/etc/modprobe.d/zfs.conf`, read back | +| `module-options` | `name=digits` from a fixed list | the options file; a live parameter also live, others at the next load of the kernel module | +| `hostid` | eight hexadecimal digits | `/etc/hostid`, by `zgenhostid -f`, only when the pool's label holds 0 or this value | +| `zed`, `zed-handlers` | `on` or `off`; `all-syslog`, `resilver_finish-start-scrub` | the module's own ZED folder, without links | +| `trim-schedule`, `trim-day`, `trim-time` | `none`, `weekly` or `monthly` with a day and a time | the scheduled run | +| `cache-devices`, `log-devices`, `log-mirror` | names under `/dev/disk/by-id` | the apply, after its checks | +| `pool-properties`, `dataset-properties` | the listed properties | the apply, read back | +| `snapshots` | `{"none": true}` or keep counts `hourly`, `daily`, `weekly`, `monthly`, 0 to 168 | the scheduled run | + +A value of another form is refused before anything is written or run (`config.refused`); one refused value refuses the whole +config. A keep count of 0 is that kind's none: none is taken, and its existing scheduled ZFS snapshots are kept with a warning. +A trim schedule of `weekly` or `monthly` without a `trim-day` or `trim-time` is refused. Changing `hostid` needs a check of the +pool's import at the next boot; the module never reboots. + +## The apply, the run, the keeper and the follower + +The module writes `/etc/zfs-tools/config.json` (all its values as one JSON object). The apply (`zfs-tools apply`, root, +run-once, started again by the config file's change) converges the node under an exclusive lock, waits (`config.waiting`) while a scrub +or resilver runs or before the pool is imported, and exits non-zero only when a command it ran failed. The event follower +(`zfs-tools events`, root) starts the apply at its own start and when a scrub or resilver ends or a declared device appears, and keeps the +findings of a pool or device not ONLINE in its state `findings`. The scheduled run (`zfs-tools run`, root, every quarter of an hour) takes +the same lock, starts a due scrub or trim and returns without waiting, takes and removes the scheduled ZFS snapshots and acts on a person's +hand acts. The three root processes hold no bus account: the keeper (`zfs-tools keeper`, the module's user `zfs-keeper`) holds the +module's one, and serves a unix socket only root may open, with a closed list of calls: emit an event of the module, put, delete, get or list a key of the states `applied`, +`schedule` and `findings`, and invoke `zfs_snapshots_record`, `zfs_snapshots_schedule` or the service manager's `start` of the module's two +units. With the keeper not answering they act as with the bus down and warn. + +## Scheduled ZFS snapshots + +Each kind with a keep count above 0 is due at the first run of the hour, day, Monday or month, against the ledger's last entry. For every +filesystem of the pool the run takes `zfs snapshot -- @mesh--`, never recursive, sets `mesh:schedule`, and records +`taken` in the ledger `snapshots` (one instance per dataset and kind, its state a map keyed by the ZFS snapshot's name) through the tool +`zfs_snapshots_record`. It takes none it cannot record, none with the ledger unreadable or full, and none while the pool is more than 80 % full. +It removes the oldest past a keep count only by their own record: name, `mesh:schedule` and ledger entry fit, no hold and no clone. +A snapshot taken by hand (`zfs_snapshot`, which refuses a `mesh-` name, event `snapshot.taken.person`) is never removed. + ## Tools -The tools run as the operator account. What ZFS lets any account read runs as that account. What ZFS -keeps for root goes through `sudo -n` with a fixed argument list, never a shell: events, history, the -snapshot and the scrub acts. Sizes are whole bytes, read under `-p`, except where status prints human sizes -even under `-p` (a scan's sizes). Each answer also says its findings in lines under `said`. +The tools run as the operator account. What ZFS keeps for root goes through `sudo -n` with a fixed argument list, never a shell. A command that +runs out of time is ended with everything it started. -A command that runs out of time is ended with everything it started. It runs in a process group of its -own. At the deadline the group gets SIGTERM, which sudo passes on to the command it runs as root, and -after five seconds SIGKILL. An act that ran out of time is answered from what the machine shows afterwards, -or as "outcome unknown". +| tool | what | +|---|---| +| `zfs_datasets`, `zfs_snapshots`, `zfs_pools`, `zfs_space`, `zfs_pool_layout`, `zfs_scrub`, `zfs_io`, `zfs_arc`, `zfs_events`, `zfs_properties`, `zfs_health` | the reads (below) | +| `zfs_snapshot` | takes `dataset@name`, a person's act with why | +| `zfs_scrub_start`, `zfs_scrub_pause`, `zfs_scrub_stop` | a person's scrub acts | +| `zfs_config` | every value with its config value, what runs, how it stands, the package's files, and what the module says before it leaves | +| `zfs_config_apply` | converge now, as the apply does | +| `zfs_snapshots_schedule` | the keep counts, next moments, what the next run would remove, and the ledger | +| `zfs_snapshots_run_now`, `zfs_snapshots_prune_now`, `zfs_snapshots_pause`, `zfs_snapshots_resume` | a person's acts on the scheduled ZFS snapshots; the hand acts wait up to 30 seconds for their event, else answer "started" | +| `zfs_module_events` | the module's own events, newest first (`zfs_events` reads ZFS's) | +| `zfs_snapshots_check` | the ledger against what ZFS holds; changes nothing | +| `zfs_findings` | the follower's findings | +| `zfs_snapshots_record` | the run's own record of a `taken` or `removed` ZFS snapshot | -| tool | what | replaces | -|---|---|---| -| `zfs_datasets` | Every filesystem and volume: mountpoint, used, available, referenced, compression ratio, sharenfs, and the number of snapshots with the space they hold together | `zfs list` | -| `zfs_snapshots` | The snapshots of one `dataset` (optional), oldest first: creation time, used (the space only that snapshot holds) and referenced | `zfs list -t snapshot` | -| `zfs_snapshot` | Takes `dataset@name` (see below) | `zfs snapshot` | -| `zfs_pools` | Each pool's state with status's reason and action, space, capacity and fragmentation %, device error count, data errors and scans | `zpool list`, `zpool status -x` | -| `zfs_space` | Each pool's space, then every dataset, biggest first, with what uses its space, quotas and reservations, compression and the savings | `zfs list -o space`, `zpool list -v` | -| `zfs_pool_layout` | Vdevs and disks, with state, error counts and status's note; log, cache, special and dedup devices; spares | `zpool status`, `zpool status -v` | -| `zfs_scrub` | Each pool's scans: its scrub or error scrub, and its resilver (pool-wide, or each vdev's rebuild), with progress and time to go while one runs; the days since the last finished scrub; and the module's schedule with its next run. `history: true` also reads the pool's history | `zpool history -i` | -| `zfs_scrub_start` | Starts a scrub of `pool`, or resumes a paused one, with `why` | `zpool scrub` | -| `zfs_scrub_pause` | Pauses one, keeping its place, with `why` | `zpool scrub -p` | -| `zfs_scrub_stop` | Stops one, losing its place, with `why` | `zpool scrub -s` | -| `zfs_io` | A sample of `seconds` (default 3, from 1 to 10): operations and bytes per second per pool, vdev and disk | `zpool iostat -v` | -| `zfs_arc` | ARC size, target, minimum and maximum, hit ratio since boot, and L2ARC where present | `arcstat`, `arc_summary` | -| `zfs_events` | Recent events, newest first, as root. By default it shows only errors and changes of state, and `all: true` shows every event | `zpool events -v` | -| `zfs_properties` | Every property of one `dataset` with its source. The value of `keylocation` is withheld | `zfs get all` | -| `zfs_health` | The health check (below) | | +When the module leaves a node, the next load of the kernel module uses ZFS's defaults, `zfs-import-cache`, `zfs-mount` and `zfs.target` stay +running, `/etc/hostid` stays, and pool and dataset properties and devices stay as the last apply set them. ## The snapshot @@ -75,17 +115,17 @@ It runs `sudo -n zfs snapshot -- @`. It answers from `zfs list` a it ran: the snapshot made, its creation time, and any other snapshot that came or went during the call. It also gives the dataset's free space and the pool's capacity. **From then on, the snapshot holds every block of the dataset that is deleted or overwritten, so that space is not freed until the snapshot is destroyed by -hand.** No tool of the mesh destroys a snapshot. +hand.** The scheduled run destroys only the scheduled ZFS snapshots it took and recorded, by their own record. -The act is recorded as the event `snapshot.taken`, and ZFS keeps the command in the pool's own history. +The act is recorded as the event `snapshot.taken.person`, and ZFS keeps the command in the pool's own history. ## The scrub schedule -The module owns the schedule of the pool its `scrub-pool` setting names. +The module owns the schedule of the pool its `pool` setting names (before this version `scrub-pool`, which mesh-config renames). | setting | default | what | |---|---|---| -| `scrub-pool` | none: the operator's value | the pool to scrub | +| `pool` | none: the operator's value | the pool to scrub | | `scrub-cadence` | `weekly` | `weekly`, or `monthly` (the first given weekday of each month) | | `scrub-day` | `Mon` | `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, `Sat` or `Sun`, nothing else | | `scrub-time` | `00:00` | `HH:MM` from `00:00` to `23:59`, in the machine's time zone | @@ -93,17 +133,13 @@ The module owns the schedule of the pool its `scrub-pool` setting names. The defaults are what the distribution's weekly timer did on the home server before the module, so taking it over does not move the schedule. The operator chose monthly for the home server. -**The scheduled run.** The node-engine runs the module's scheduled process `zfs-mesh-scrub` (`zfs-tools -scrub-run`, as root) from its own timer `zfs-mesh-scrub.timer`, every quarter of an hour. The run reads the -configuration the mesh writes from the settings (`/etc/zfs-tools/scrub.conf`). It finds the slot, which is -the last scheduled time not after now. If the slot is not yet served, the run acts as upstream's -`zfs-scrub@.service` does: +**The scheduled run.** The node-engine runs the module's scheduled process `zfs-mesh-run` (`zfs-tools run`, as root) from its +own timer `zfs-mesh-run.timer`, every quarter of an hour. The run reads `/etc/zfs-tools/config.json`. It finds the slot, which +is the last scheduled time not after now. If the slot is not yet served, the run starts the scrub with `zpool scrub -- ` +and returns at once: it never waits for it, and the event follower hears the scrub end. If a scrub or resilver runs, or the pool +is not imported, the run does nothing and says `run.skipped`. -- if a scrub is running, it waits for it with `zpool wait -t scrub -- `; -- otherwise it starts one with `zpool scrub -- ` and waits for it the same way; -- if the start is refused because a scrub has just begun, it waits for that one. - -It never pauses or stops a scrub. The unit has no ExecStop, so stopping the unit ends only the wait. A +It never pauses or stops a scrub. A resilver that is running defers the scrub to the next run after it. The served slot is kept in `/var/lib/zfs-scrub/scrub-served.json`, in a folder of root's alone (0700), apart from the account's state. The run writes it only into a real folder of its own and never through a link: a new file opened with @@ -119,7 +155,7 @@ the names of the two distribution timers. The controller fills it there and refu digits, `:`, `_`, `.` and `-` (mesh-controller #171). **Exactly one schedule per pool.** The distribution's `zfs-scrub-weekly@.timer` and -`zfs-scrub-monthly@.timer` are held stopped and disabled. They come after the scheduled run in the +`zfs-scrub-monthly@.timer` are held stopped and disabled, and the trim timers with them while `trim-schedule` is set. They come after the scheduled run in the declaration, so a run that cannot be installed stops the apply before the timer that scrubs the pool today is touched. On the home server the weekly one was enabled by hand, outside the mesh. A scrub already running is not stopped by this; only its timer is. @@ -161,7 +197,7 @@ The module runs two long-running processes of its own binary, and the scheduled their health changes. - **The event follower** (`zfs-tools events`) runs as root and follows `zpool events -f`. It is described in the next section. -- **The scheduled run** (`zfs-tools scrub-run`) is not long-running and has no health check of its own; +- **The scheduled run** (`zfs-tools run`) is not long-running and has no health check of its own; `zfs_health` says when it failed or did not start a due scrub. `zfs_health` is **unhealthy** for a fault now: @@ -216,12 +252,17 @@ The tests cover: paused scrubs) and `iostat -H` without section lines; - a timed-out command ended with its children; - the snapshot's refusals, read-back, space statement and unknown outcome; -- the scrub acts, the schedule's slots, and the scheduled run (start, wait, never pause, defer to a +- the scrub acts, the schedule's slots, and the scheduled run (start and return, never pause, defer to a resilver, refuse an invalid setting); +- the config's parse, one table per value; each convergence plan (properties, devices, the node's ZFS id, the ARC); the lock; the + scheduled ZFS snapshots' taking and removal; the hand acts; the keeper's closed list of calls and its socket; the tools; the health; - the follower's cursor, dropped events, kill before wait, and exit after repeated failures; - the health verdicts, past against new errors; - a machine without ZFS; -- the manifest against the tools the bundle serves. +- the manifest (values, files, units, order, bus state, keeper) against the tools the bundle serves and the call-site test. + +The module adds no integration test (its test budget is 500 unit tests and 50 integration tests): the main design's live tests stand for +the module whole. ## Files it does not know @@ -231,3 +272,6 @@ Each directory or archive this module declares says how it treats a file it did - `/run/zfs-events`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `/etc/zfs-tools`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `/var/lib/zfs-scrub`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `/run/zfs-tools`: everything in it is the program's (the lock); the mesh moves nothing out. +- the keeper's folder: everything in it is the program's (its socket and its credential). +- `/etc/zfs/mesh-zed.d`: the module's own, while `zed` is on. diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index ea7021441..4d19bf8ad 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -540,3 +540,19 @@ func TestTheManifestsBusStateKeeperAndTools(t *testing.T) { } } } + +// §31, the test budget: the module stays within 500 unit tests and adds no integration test (its live tests are the main design's). +func TestTheModuleStaysWithinItsTestBudget(t *testing.T) { + files, _ := filepath.Glob("*_test.go") + n := 0 + for _, f := range files { + b, _ := os.ReadFile(f) + if strings.Contains(string(b), "//go:build integration") { + t.Errorf("%s is an integration test: the module adds none", f) + } + n += len(regexp.MustCompile(`(?m)^func Test`).FindAllString(string(b), -1)) + } + if n > 500 { + t.Errorf("%d unit tests, over the budget of 500", n) + } +} From 605c1b1ca021a32b64c566ebf99ab7c4b15481a0 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:09:16 +0100 Subject: [PATCH 21/35] Z19 the budget test does not match its own text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §31 --- modules/zfs/cmd/zfs-tools/manifest_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 4d19bf8ad..cac180233 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -547,7 +547,7 @@ func TestTheModuleStaysWithinItsTestBudget(t *testing.T) { n := 0 for _, f := range files { b, _ := os.ReadFile(f) - if strings.Contains(string(b), "//go:build integration") { + if strings.Contains(string(b), "//go:"+"build integration") { t.Errorf("%s is an integration test: the module adds none", f) } n += len(regexp.MustCompile(`(?m)^func Test`).FindAllString(string(b), -1)) From 008e02afbd75f2b25a2c508422d260662ef02225 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:28:51 +0100 Subject: [PATCH 22/35] zfs: take out the events stream's max-mib, a bound no design line gives (review finding 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §25 --- modules/zfs/module.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/zfs/module.json b/modules/zfs/module.json index fd2b98256..e4c93d234 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -33,8 +33,7 @@ "run.skipped", "scrub.requested" ], - "max-age-days": 30, - "max-mib": 16 + "max-age-days": 30 } }, { From 5f372ac63a0b920890562efd95f4f639d766c2c3 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:28:51 +0100 Subject: [PATCH 23/35] zfs: read an unwritten ledger instance as state null and sequence 0, pass every error up (review finding 2, A-40) Builds: design 42 zfs-owns-zfs A-40 --- modules/zfs/cmd/zfs-tools/bus.go | 36 +++++++++---------- .../zfs/cmd/zfs-tools/ledger_absent_test.go | 21 +++++++++++ 2 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/ledger_absent_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 347e0d57e..5aabd9049 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -162,15 +162,22 @@ func (busLedger) Apply(instance, operation string, body any, call string) error return err } -// absentInstance says whether a ledger get said the instance is not there, which an instance never written is. -func absentInstance(err error) bool { - s := strings.ToLower(err.Error()) - for _, w := range []string{"not found", "no instance", "no such", "does not exist", "unknown instance", "not kept"} { - if strings.Contains(s, w) { - return true - } +// takenOf reads a ledger get's answer: an instance never written answers state null and sequence 0, which is empty (A-40). +func takenOf(raw []byte) (struct { + Taken map[string]LedgerEntry `json:"taken"` +}, error) { + var body struct { + Taken map[string]LedgerEntry `json:"taken"` } - return false + var st buscalls.InstanceState + if err := json.Unmarshal(raw, &st); err != nil { + return body, err + } + if st.Sequence == 0 && (len(st.State) == 0 || string(st.State) == "null") { + return body, nil + } + err := json.Unmarshal(st.State, &body) + return body, err } func (busLedger) Entries(datasets []string) ([]LedgerEntry, error) { @@ -179,19 +186,10 @@ func (busLedger) Entries(datasets []string) ([]LedgerEntry, error) { for _, kind := range keepKinds { raw, err := stdio.BusState(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: LedgerName, Instance: LedgerInstance(kind, ds)}) if err != nil { - if absentInstance(err) { - continue - } return nil, err } - var st buscalls.InstanceState - if err := json.Unmarshal(raw, &st); err != nil { - return nil, err - } - var body struct { - Taken map[string]LedgerEntry `json:"taken"` - } - if err := json.Unmarshal(st.State, &body); err != nil { + body, err := takenOf(raw) + if err != nil { return nil, err } for _, name := range sortedKeys(body.Taken) { diff --git a/modules/zfs/cmd/zfs-tools/ledger_absent_test.go b/modules/zfs/cmd/zfs-tools/ledger_absent_test.go new file mode 100644 index 000000000..e9681d22e --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/ledger_absent_test.go @@ -0,0 +1,21 @@ +package main + +import "testing" + +func TestTakenOfLedgerAnswers(t *testing.T) { + for _, c := range []struct { + name, raw string + want int + err bool + }{ + {"never written", `{"instance":"x","state":null,"sequence":0}`, 0, false}, + {"written", `{"instance":"x","state":{"taken":{"a":{}}},"sequence":3}`, 1, false}, + {"bad state", `{"instance":"x","state":"oops","sequence":3}`, 0, true}, + {"bad answer", `nope`, 0, true}, + } { + b, err := takenOf([]byte(c.raw)) + if (err != nil) != c.err || len(b.Taken) != c.want { + t.Errorf("%s: %v %d", c.name, err, len(b.Taken)) + } + } +} From e79392d6667aaaae02671b3c167cc647d77d5e44 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:29:15 +0100 Subject: [PATCH 24/35] zfs: a follower reaches the keeper's socket on every call, so a keeper up later is used (A-37, reviewer's note) Builds: design 42 zfs-owns-zfs A-37 --- modules/zfs/cmd/zfs-tools/bus.go | 9 +++----- modules/zfs/cmd/zfs-tools/keeper_test.go | 26 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 5aabd9049..7d0654061 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -36,7 +36,7 @@ func (b busStates) Get(state, key string) (json.RawMessage, bool, error) { } // JoinBus gives a root process the module's events and states through the keeper's socket (A-37): the process holds no -// bus account of its own. Without the socket the machine keeps none, and what it changes is said as a warning (§18: +// bus account of its own. Without the socket variable the machine keeps none, and what it changes is said as a warning (§18: // with the bus down the apply converges from the config file it last got). It answers how to leave the bus. func (m *Machine) JoinBus() func() { sock := strings.TrimSpace(os.Getenv(KeeperSocketEnv)) @@ -44,12 +44,9 @@ func (m *Machine) JoinBus() func() { m.Warnings = append(m.Warnings, KeeperSocketEnv+" is not set, so the keeper is not reached and the bus is taken as down") return func() {} } + // Every call reaches the socket anew, so a keeper that comes up after this process is used from then on; one that + // is down at a call answers that call with an error, never the bus taken as down for the process's life. c := keeperClient{sock} - // The keeper answering is what the bus being up means: one call says it before the process relies on it. - if _, err := c.Keys(StateSchedule); err != nil { - m.Warnings = append(m.Warnings, err.Error()+": the bus is taken as down") - return func() {} - } m.Emit = c.emit m.States = c m.Findings = meshconditionsclient.FindingWriter{State: findingState{c}} diff --git a/modules/zfs/cmd/zfs-tools/keeper_test.go b/modules/zfs/cmd/zfs-tools/keeper_test.go index 9c5cbb2cd..fda12c202 100644 --- a/modules/zfs/cmd/zfs-tools/keeper_test.go +++ b/modules/zfs/cmd/zfs-tools/keeper_test.go @@ -106,14 +106,36 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. if _, left := st["schedule/x"]; left { t.Errorf("a refused peer's call was read") } - // No keeper: the machine keeps no bus and says so. - t.Setenv(KeeperSocketEnv, filepath.Join(dir, "none.sock")) + // No socket variable: the machine keeps no bus and says so. + t.Setenv(KeeperSocketEnv, "") m := newFake(t).machine() m.Emit = nil m.JoinBus() if m.Emit != nil || m.States != nil || len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "bus is taken as down") { t.Errorf("no keeper: %+v", m.Warnings) } + // A keeper not yet up: the process joins, its calls fail, and the same process uses the keeper once it is up. + late := filepath.Join(dir, "late.sock") + t.Setenv(KeeperSocketEnv, late) + ml := newFake(t).machine() + ml.JoinBus() + if ml.States == nil { + t.Fatalf("a keeper not up yet must not take the bus as down: %+v", ml.Warnings) + } + if _, err := ml.States.Keys(StateSchedule); err == nil { + t.Errorf("no keeper up, yet a call answered") + } + ll, err := Listen(late) + if err != nil { + t.Fatal(err) + } + lctx, lcancel := context.WithCancel(context.Background()) + defer lcancel() + go (&Keeper{Emit: func(string, any) error { return nil }, States: memStates{}, PeerOf: func(net.Conn) (int, error) { return 0, nil }, + Call: func(string, any) (json.RawMessage, error) { return nil, nil }}).Serve(lctx, ll) + if _, err := ml.States.Keys(StateSchedule); err != nil { + t.Errorf("a keeper that came up later is not used: %v", err) + } t.Setenv(KeeperSocketEnv, sock) uid = 0 m2 := newFake(t).machine() From 87f8c30a096387dd3a9a6d18e35cfd78a52ac64e Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:31:08 +0100 Subject: [PATCH 25/35] zfs: the keeper publishes events with an acknowledged JetStream publish on mesh.bus.zfs.events. (review finding 1, A-41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs A-41, §25 --- modules/zfs/cmd/zfs-tools/bus.go | 35 ++++++++++++++++++- .../zfs/cmd/zfs-tools/publish_event_test.go | 30 ++++++++++++++++ modules/zfs/go.mod | 2 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/publish_event_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index 7d0654061..af6ffb6c5 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -13,6 +13,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // busStates is the module's states as this process reaches them (its bus account's `state`). @@ -73,10 +75,41 @@ func RunKeeper(ctx context.Context) error { if err != nil { return err } - k := &Keeper{Emit: func(kind string, body any) error { return c.Emit(kind, body, stdio.EmitOptions{}) }, States: busStates{c}, Call: c.Call} + _, js, err := acct.Dial() + if err != nil { + return err + } + k := &Keeper{Emit: func(kind string, body any) error { return publishEvent(js, acct, kind, body) }, States: busStates{c}, Call: c.Call} return k.Serve(ctx, l) } +// eventPublisher is the part of JetStream the keeper publishes with. +type eventPublisher interface { + PublishMsg(ctx context.Context, m *nats.Msg, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) +} + +// publishEvent puts one event on the module's events stream as a JetStream publish the bus acknowledges, on the +// subject the stream captures, mesh.bus..events. (A-41); never process.Conn.Emit, whose subject the +// keeper's account is not granted. The message id is a random one, as before. +func publishEvent(js eventPublisher, acct process.Account, kind string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + id := fmt.Sprintf("%d-%d", time.Now().UnixNano(), os.Getpid()) + msg := nats.NewMsg(acct.BusSubject(EventsStream, kind)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", acct.Module) + msg.Header.Set("x-node", acct.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id)) + return err +} + // toolStates is the module's states as the tools bundle reaches them, through the tool runner. type toolStates struct{} diff --git a/modules/zfs/cmd/zfs-tools/publish_event_test.go b/modules/zfs/cmd/zfs-tools/publish_event_test.go new file mode 100644 index 000000000..df3177b67 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/publish_event_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "testing" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" +) + +type recordingPublisher struct{ msgs []*nats.Msg } + +func (r *recordingPublisher) PublishMsg(_ context.Context, m *nats.Msg, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + r.msgs = append(r.msgs, m) + return &jetstream.PubAck{}, nil +} + +// A-41, review finding 1: the keeper publishes on the subject the module's events stream captures. +func TestTheKeeperPublishesOnTheEventsStreamsSubject(t *testing.T) { + p := &recordingPublisher{} + acct := process.Account{Node: "n1", Module: "zfs"} + if err := publishEvent(p, acct, EventApplied, map[string]string{"key": "x"}); err != nil { + t.Fatal(err) + } + if len(p.msgs) != 1 || p.msgs[0].Subject != "mesh.bus.zfs.events."+EventApplied || p.msgs[0].Header.Get("x-event-id") == "" { + t.Errorf("%+v", p.msgs) + } +} diff --git a/modules/zfs/go.mod b/modules/zfs/go.mod index 8b1859664..9078972fa 100644 --- a/modules/zfs/go.mod +++ b/modules/zfs/go.mod @@ -11,7 +11,7 @@ require ( require ( github.com/klauspost/compress v1.20.1 // indirect - github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.57.0 // indirect From 2d592c63c16ca63a3dd97a1cda656bfc8e5d564f Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:45:19 +0100 Subject: [PATCH 26/35] zfs: a keeper that does not answer is warned once, and RunKeeper closes its second connection (review round 2 finding 2, A-37) A failed keeper call in a root process (hand acts, forget, Paused, every call) now adds one warning that the bus is taken as down, instead of being dropped silently. RunKeeper closes the connection acct.Dial opens when the keeper stops; the message id's comment says it is nanotime-pid. Builds: design 42 zfs-owns-zfs A-37 --- modules/zfs/cmd/zfs-tools/bus.go | 13 ++++++++--- modules/zfs/cmd/zfs-tools/keeper.go | 26 ++++++++++++++++------ modules/zfs/cmd/zfs-tools/keeper_test.go | 28 +++++++++++++++++++++++- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index af6ffb6c5..b5da08576 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -48,7 +48,13 @@ func (m *Machine) JoinBus() func() { } // Every call reaches the socket anew, so a keeper that comes up after this process is used from then on; one that // is down at a call answers that call with an error, never the bus taken as down for the process's life. - c := keeperClient{sock} + // A call the keeper does not answer is said once, as a warning: the root process then acts as with the bus down (A-37). + var once sync.Once + c := keeperClient{sock: sock, down: func(err error) { + once.Do(func() { + m.Warnings = append(m.Warnings, "the bus is taken as down: "+err.Error()) + }) + }} m.Emit = c.emit m.States = c m.Findings = meshconditionsclient.FindingWriter{State: findingState{c}} @@ -75,10 +81,11 @@ func RunKeeper(ctx context.Context) error { if err != nil { return err } - _, js, err := acct.Dial() + nc, js, err := acct.Dial() if err != nil { return err } + defer nc.Close() k := &Keeper{Emit: func(kind string, body any) error { return publishEvent(js, acct, kind, body) }, States: busStates{c}, Call: c.Call} return k.Serve(ctx, l) } @@ -90,7 +97,7 @@ type eventPublisher interface { // publishEvent puts one event on the module's events stream as a JetStream publish the bus acknowledges, on the // subject the stream captures, mesh.bus..events. (A-41); never process.Conn.Emit, whose subject the -// keeper's account is not granted. The message id is a random one, as before. +// keeper's account is not granted. The message id (nanotime-pid) is the time in nanoseconds and the process id, so two events of one process never share one. func publishEvent(js eventPublisher, acct process.Account, kind string, body any) error { data, err := json.Marshal(body) if err != nil { diff --git a/modules/zfs/cmd/zfs-tools/keeper.go b/modules/zfs/cmd/zfs-tools/keeper.go index fc63abf7e..0ffa12d79 100644 --- a/modules/zfs/cmd/zfs-tools/keeper.go +++ b/modules/zfs/cmd/zfs-tools/keeper.go @@ -239,26 +239,38 @@ func Listen(sock string) (net.Listener, error) { } // keeperClient is a root process's way to the bus: calls on the keeper's socket. -type keeperClient struct{ sock string } +type keeperClient struct { + sock string + // down, when set, is told of a call the keeper did not answer (not of an answer that is an error). + down func(error) +} func (c keeperClient) do(r KeeperRequest) (KeeperAnswer, error) { + a, err, unanswered := c.exchange(r) + if unanswered && c.down != nil { + c.down(err) + } + return a, err +} + +// exchange is one call on the socket; unanswered says the keeper gave no answer (as opposed to an answer that is an error). +func (c keeperClient) exchange(r KeeperRequest) (a KeeperAnswer, err error, unanswered bool) { conn, err := net.DialTimeout("unix", c.sock, 5*time.Second) if err != nil { - return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err) + return KeeperAnswer{}, fmt.Errorf("the keeper does not answer on %s: %v", c.sock, err), true } defer conn.Close() _ = conn.SetDeadline(time.Now().Add(KeeperTimeout)) if err := json.NewEncoder(conn).Encode(r); err != nil { - return KeeperAnswer{}, err + return KeeperAnswer{}, err, true } - var a KeeperAnswer if err := json.NewDecoder(conn).Decode(&a); err != nil { - return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err) + return KeeperAnswer{}, fmt.Errorf("the keeper's answer: %v", err), true } if a.Error != "" { - return a, errors.New(a.Error) + return a, errors.New(a.Error), false } - return a, nil + return a, nil, false } func (c keeperClient) emit(kind string, body any) error { diff --git a/modules/zfs/cmd/zfs-tools/keeper_test.go b/modules/zfs/cmd/zfs-tools/keeper_test.go index fda12c202..c27ee3e72 100644 --- a/modules/zfs/cmd/zfs-tools/keeper_test.go +++ b/modules/zfs/cmd/zfs-tools/keeper_test.go @@ -86,7 +86,7 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. ctx, stop := context.WithCancel(context.Background()) defer stop() go k.Serve(ctx, l) - c := keeperClient{sock} + c := keeperClient{sock: sock} if err := c.Put(StateSchedule, "snapshots", map[string]string{"by": "x"}); err != nil { t.Fatal(err) } @@ -144,3 +144,29 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. t.Errorf("a keeper answering: the bus is up") } } + +// A-37: a root process whose keeper does not answer acts as with the bus down and says so, once, in a warning, for the +// hand acts, the forgetting of applied keys and the pause alike. +func TestAKeeperThatDoesNotAnswerIsWarnedOnce(t *testing.T) { + dir, err := os.MkdirTemp("/var/tmp", "keeper-down") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + t.Setenv(KeeperSocketEnv, filepath.Join(dir, "none.sock")) + m := newFake(t).machine() + m.JoinBus() + if got := m.handActs(); got != nil { + t.Errorf("hand acts with no keeper: %v", got) + } + if len(m.Warnings) != 1 || !strings.Contains(m.Warnings[0], "bus is taken as down") { + t.Fatalf("hand acts: %v", m.Warnings) + } + m.forget(&Config{}) + if m.Paused() { + t.Errorf("paused with no keeper") + } + if len(m.Warnings) != 1 { + t.Errorf("said more than once: %v", m.Warnings) + } +} From 1ff904f10107e4581b50c6a25761ba42602fd29a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:50:27 +0100 Subject: [PATCH 27/35] zfs: the tools say events with mesh/stream.publish and a hand act hears its event on the events stream (A-42, A-12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tools say every event of §25 (snapshot.taken.person, scrub.requested and the others) with mesh/stream.publish on the stream events, never stdio.Emit; the hand acts' 30 s wait reads the mark (stream.fill) before the act starts, then stream.read from the next sequence, for at most HandWait. Builds: design 42 zfs-owns-zfs §25, §24, lead answers A-42, A-12 --- modules/zfs/cmd/zfs-tools/bus.go | 106 ++++++++++-------- modules/zfs/cmd/zfs-tools/bus_stream_test.go | 112 +++++++++++++++++++ modules/zfs/cmd/zfs-tools/hand.go | 8 +- modules/zfs/cmd/zfs-tools/hand_test.go | 15 ++- modules/zfs/cmd/zfs-tools/machine.go | 6 +- modules/zfs/cmd/zfs-tools/main.go | 5 +- 6 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/bus_stream_test.go diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index b5da08576..d1f072b82 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -134,55 +134,73 @@ func (toolStates) Get(state, key string) (json.RawMessage, bool, error) { return e.Value, true, nil } -// eventHub hands the module's events, heard once through the tool runner, to the hand acts waiting for one. -type eventHub struct { - mu sync.Mutex - started bool - waiters map[int]hubWaiter - next int -} - -type hubWaiter struct { - kinds []string - ch chan map[string]any -} - -func (h *eventHub) await(kinds []string, bound time.Duration) (map[string]any, bool) { - h.mu.Lock() - if h.waiters == nil { - h.waiters = map[int]hubWaiter{} +// toolEmit says one event of the module on its own events stream, by the tool runner's mesh/stream.publish (A-42): the +// runner publishes it acknowledged on the stream's subject plus the kind, the id its message id. Never stdio.Emit, which +// publishes on the module's old event subjects. The id (nanotime-pid) is the time in nanoseconds and the process id, so +// two events of one process never share one. +func toolEmit(ask func(method string, params any) (json.RawMessage, error)) func(kind string, body any) error { + return func(kind string, body any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + _, err = ask(buscalls.StreamPublish, buscalls.StreamPublishRequest{Stream: EventsStream, Event: kind, Body: raw, + ID: fmt.Sprintf("%d-%d", time.Now().UnixNano(), os.Getpid())}) + return err } - h.next++ - id, w := h.next, hubWaiter{kinds, make(chan map[string]any, 1)} - h.waiters[id] = w - first := !h.started - h.started = true - h.mu.Unlock() - defer func() { h.mu.Lock(); delete(h.waiters, id); h.mu.Unlock() }() - if first { - _ = stdio.Subscribe("#", func(e stdio.Envelope) error { - var body map[string]any - _ = json.Unmarshal(e.Body, &body) - h.mu.Lock() - defer h.mu.Unlock() - for _, w := range h.waiters { - for _, k := range w.kinds { - if e.Key == k || strings.HasSuffix(e.Key, "."+k) { - select { - case w.ch <- body: - default: +} + +// HandPoll is how long a hand act's wait lets pass between two reads of the events stream. +const HandPoll = 500 * time.Millisecond + +// streamHear lets a hand act hear its act's event by reading the module's own events stream with the calls a bundle has +// (A-12, A-42): the stream's last sequence is read before the act starts, then the stream from the next sequence on. +type streamHear struct { + bus func(method string, params any) (json.RawMessage, error) + now func() time.Time + sleep func(time.Duration) +} + +// await marks the stream's last sequence and answers the wait for an event of one of kinds after it. A stream that +// cannot be read answers a wait that sees nothing, so the act is answered "started". +func (h streamHear) await(kinds []string) func(bound time.Duration) (map[string]any, bool) { + raw, err := h.bus(buscalls.StreamFill, buscalls.StreamRequest{Stream: EventsStream}) + var fill buscalls.StreamFilled + if err == nil { + err = json.Unmarshal(raw, &fill) + } + if err != nil { + return func(time.Duration) (map[string]any, bool) { return nil, false } + } + from := fill.Last + 1 + return func(bound time.Duration) (map[string]any, bool) { + deadline := h.now().Add(bound) + for { + raw, err := h.bus(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: EventsStream, From: from, Limit: 500}) + if err == nil { + var page buscalls.StreamPage + if json.Unmarshal(raw, &page) == nil { + for _, msg := range page.Messages { + from = msg.Sequence + 1 + for _, k := range kinds { + if msg.Subject == k || strings.HasSuffix(msg.Subject, "."+k) { + var body map[string]any + _ = json.Unmarshal(msg.Body, &body) + return body, true + } } } + if page.Next != 0 { + from = page.Next + continue + } } } - return nil - }) - } - select { - case ev := <-w.ch: - return ev, true - case <-time.After(bound): - return nil, false + if !h.now().Before(deadline) { + return nil, false + } + h.sleep(HandPoll) + } } } diff --git a/modules/zfs/cmd/zfs-tools/bus_stream_test.go b/modules/zfs/cmd/zfs-tools/bus_stream_test.go new file mode 100644 index 000000000..983b348ec --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/bus_stream_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// fakeStream answers a bundle's calls on its events stream from a list of messages and a clock the wait advances. +type fakeStream struct { + msgs []buscalls.StreamMessage + asked []string + reads []uint64 + now time.Time + fillErr error + // at maps a read count to messages added before that read (an event that comes while the tool waits). + arrive map[int]buscalls.StreamMessage +} + +func (f *fakeStream) ask(method string, params any) (json.RawMessage, error) { + f.asked = append(f.asked, method) + switch method { + case buscalls.StreamFill: + if f.fillErr != nil { + return nil, f.fillErr + } + last := uint64(0) + if n := len(f.msgs); n > 0 { + last = f.msgs[n-1].Sequence + } + return json.Marshal(buscalls.StreamFilled{First: 1, Last: last}) + case buscalls.StreamRead: + r := params.(buscalls.StreamReadRequest) + f.reads = append(f.reads, r.From) + if m, ok := f.arrive[len(f.reads)]; ok { + m.Sequence = uint64(len(f.msgs) + 1) + f.msgs = append(f.msgs, m) + } + page := buscalls.StreamPage{Messages: []buscalls.StreamMessage{}} + for _, m := range f.msgs { + if m.Sequence >= r.From { + page.Messages = append(page.Messages, m) + } + } + return json.Marshal(page) + case buscalls.StreamPublish: + return json.Marshal(buscalls.StreamPublished{Sequence: 9}) + } + return nil, errors.New("unexpected " + method) +} + +func (f *fakeStream) hear() streamHear { + return streamHear{bus: f.ask, now: func() time.Time { return f.now }, sleep: func(d time.Duration) { f.now = f.now.Add(d) }} +} + +// A-12, A-42: a hand act's wait marks the events stream's last sequence before the act starts, then reads from the next +// one: an event before the mark is not the act's, one after it with an end kind is, and with none in 30 s the wait gives up. +func TestAHandWaitHearsTheEventsStreamAfterItsMark(t *testing.T) { + old := buscalls.StreamMessage{Sequence: 1, Subject: "mesh.bus.zfs.events.snapshot.taken.person", Body: json.RawMessage(`{"old":true}`)} + other := buscalls.StreamMessage{Subject: "mesh.bus.zfs.events.config.refused", Body: json.RawMessage(`{}`)} + end := buscalls.StreamMessage{Subject: "mesh.bus.zfs.events.snapshot.taken.person", Body: json.RawMessage(`{"snapshot":"tank@x"}`)} + + f := &fakeStream{msgs: []buscalls.StreamMessage{old}, now: time.Unix(0, 0), arrive: map[int]buscalls.StreamMessage{2: other, 3: end}} + wait := f.hear().await([]string{EventTakenPerson}) + if len(f.asked) != 1 || f.asked[0] != buscalls.StreamFill { + t.Fatalf("the mark was not read first: %v", f.asked) + } + ev, ok := wait(HandWait) + if !ok || ev["snapshot"] != "tank@x" { + t.Fatalf("heard %v %v; the old event or none", ev, ok) + } + if f.reads[0] != 2 { + t.Errorf("read from %d, want the sequence after the mark, 2", f.reads[0]) + } + + g := &fakeStream{msgs: []buscalls.StreamMessage{old}, now: time.Unix(0, 0)} + start := g.now + if ev, ok := g.hear().await([]string{EventTakenPerson})(HandWait); ok || ev != nil { + t.Fatalf("heard %v with nothing new on the stream", ev) + } + if g.now.Sub(start) < HandWait { + t.Errorf("gave up after %s, before the 30 s", g.now.Sub(start)) + } + + h := &fakeStream{fillErr: errors.New("no stream"), now: time.Unix(0, 0)} + if _, ok := h.hear().await([]string{EventTakenPerson})(HandWait); ok { + t.Error("heard an event from a stream that could not be read") + } +} + +// A-42: the tool says an event with mesh/stream.publish on the stream `events`, the kind as the event and a message id. +func TestAToolSaysItsEventWithStreamPublish(t *testing.T) { + var got buscalls.StreamPublishRequest + var method string + say := toolEmit(func(m string, p any) (json.RawMessage, error) { + method, got = m, p.(buscalls.StreamPublishRequest) + return json.RawMessage(`{"sequence":9}`), nil + }) + if err := say(ScrubEvent, map[string]any{"pool": "tank"}); err != nil { + t.Fatal(err) + } + if method != buscalls.StreamPublish || got.Stream != "events" || got.Event != ScrubEvent || got.ID == "" || string(got.Body) != `{"pool":"tank"}` { + t.Errorf("asked %s %+v", method, got) + } + fail := toolEmit(func(string, any) (json.RawMessage, error) { return nil, errors.New("refused") }) + if fail(ScrubEvent, nil) == nil { + t.Error("a refused publish was said done") + } +} diff --git a/modules/zfs/cmd/zfs-tools/hand.go b/modules/zfs/cmd/zfs-tools/hand.go index a85c28704..6d30d4c83 100644 --- a/modules/zfs/cmd/zfs-tools/hand.go +++ b/modules/zfs/cmd/zfs-tools/hand.go @@ -66,6 +66,10 @@ func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, if m.States == nil || m.Call == nil { return a, fmt.Errorf("this process reaches neither the module's state nor the service manager, so %s was not started", act) } + var wait func(time.Duration) (map[string]any, bool) + if m.Await != nil { + wait = m.Await(handEnds[act]) // marks the stream's last sequence before the act starts + } if err := m.States.Put(StateSchedule, act, map[string]any{"by": by, "why": why, "at": m.Now().UTC()}); err != nil { return a, fmt.Errorf("the act %s could not be written to the state %s: %v", act, StateSchedule, err) } @@ -73,8 +77,8 @@ func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, _ = m.States.Delete(StateSchedule, act) return a, fmt.Errorf("the service manager did not start %s: %v", RunService, err) } - if m.Await != nil { - if ev, ok := m.Await(handEnds[act], HandWait); ok { + if wait != nil { + if ev, ok := wait(HandWait); ok { a.Outcome, a.Event = "seen", ev a.Said = fmt.Sprintf("The act %s ran: %v.", act, ev) return a, nil diff --git a/modules/zfs/cmd/zfs-tools/hand_test.go b/modules/zfs/cmd/zfs-tools/hand_test.go index c4f685e86..39bf1344e 100644 --- a/modules/zfs/cmd/zfs-tools/hand_test.go +++ b/modules/zfs/cmd/zfs-tools/hand_test.go @@ -37,12 +37,16 @@ func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { m.States, m.Node = st, "ace" var key string var body any - m.Call = func(k string, b any) (json.RawMessage, error) { key, body = k, b; return nil, c.callErr } + var marked, started bool + m.Call = func(k string, b any) (json.RawMessage, error) { key, body, started = k, b, true; return nil, c.callErr } var bound time.Duration var kinds []string - m.Await = func(k []string, d time.Duration) (map[string]any, bool) { - kinds, bound = k, d - return c.event, c.event != nil + m.Await = func(k []string) func(time.Duration) (map[string]any, bool) { + kinds, marked = k, !started + return func(d time.Duration) (map[string]any, bool) { + bound = d + return c.event, c.event != nil + } } a, err := m.HandAct(context.Background(), c.act, "caller@node", c.why) if c.err != "" { @@ -61,6 +65,9 @@ func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { t.Errorf("%s: called %s with %v", c.name, key, body) } } + if c.err == "" && !marked { + t.Errorf("%s: the stream was not marked before the act started", c.name) + } if c.err == "" && (bound != 30*time.Second || strings.Join(kinds, ",") != strings.Join(c.waited, ",")) { t.Errorf("%s: waited %v for %v", c.name, bound, kinds) } diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index 55ad575b1..fd2e6e0c5 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -125,9 +125,9 @@ type Machine struct { Call func(key string, body any) (json.RawMessage, error) // Node is this machine's name, for a seat verb of its own. Node string - // Await waits at most the bound for an event of the module whose kind is one of kinds, and answers it; false - // when none came. - Await func(kinds []string, bound time.Duration) (map[string]any, bool) + // Await marks the module's events stream before an act starts and answers the wait for an event of the module whose + // kind is one of kinds, after the mark: at most the bound, and the event; false when none came (A-12, A-42). + Await func(kinds []string) func(bound time.Duration) (map[string]any, bool) // Findings writes the module's own state `findings` as the seat mesh-conditions' layout says (A-32). Findings views.FindingWriter // Warnings are what the apply saw that is no failure (a change the bus did not take, a parameter awaiting diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index b10224215..bfa579340 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -16,6 +16,7 @@ import ( "os" "os/signal" "syscall" + "time" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -71,8 +72,8 @@ func main() { return } m := Here() - m.Emit = func(eventType string, body any) error { return stdio.Emit(eventType, body) } - m.States, m.Call, m.Ledger, m.Await, m.Log = toolStates{}, stdio.Ask, busLedger{}, (&eventHub{}).await, busStream{} + m.Emit = toolEmit(stdio.BusState) + m.States, m.Call, m.Ledger, m.Await, m.Log = toolStates{}, stdio.Ask, busLedger{}, streamHear{bus: stdio.BusState, now: time.Now, sleep: time.Sleep}.await, busStream{} // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. if err := stdio.Serve("", Tools(m)); err != nil { fmt.Fprintln(os.Stderr, err) From 58664eee60c5804fbeb6781669acac4992cee3ae Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 01:52:50 +0100 Subject: [PATCH 28/35] zfs: the hand acts answer started at once and the manifest asks stream-publish, bus-state-calls and ledger-calls (A-43, A-44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes streamHear, HandPoll, HandWait and the stream mark; keeps toolEmit. Builds: design 42 zfs-owns-zfs §24, §25, lead answers A-43, A-44 --- modules/zfs/cmd/zfs-tools/bus.go | 54 ------------- modules/zfs/cmd/zfs-tools/bus_stream_test.go | 83 -------------------- modules/zfs/cmd/zfs-tools/hand.go | 40 +++------- modules/zfs/cmd/zfs-tools/hand_test.go | 33 ++------ modules/zfs/cmd/zfs-tools/machine.go | 3 - modules/zfs/cmd/zfs-tools/main.go | 3 +- modules/zfs/cmd/zfs-tools/tools.go | 6 +- modules/zfs/module.json | 5 +- 8 files changed, 26 insertions(+), 201 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/bus.go b/modules/zfs/cmd/zfs-tools/bus.go index d1f072b82..f6ca75d80 100644 --- a/modules/zfs/cmd/zfs-tools/bus.go +++ b/modules/zfs/cmd/zfs-tools/bus.go @@ -150,60 +150,6 @@ func toolEmit(ask func(method string, params any) (json.RawMessage, error)) func } } -// HandPoll is how long a hand act's wait lets pass between two reads of the events stream. -const HandPoll = 500 * time.Millisecond - -// streamHear lets a hand act hear its act's event by reading the module's own events stream with the calls a bundle has -// (A-12, A-42): the stream's last sequence is read before the act starts, then the stream from the next sequence on. -type streamHear struct { - bus func(method string, params any) (json.RawMessage, error) - now func() time.Time - sleep func(time.Duration) -} - -// await marks the stream's last sequence and answers the wait for an event of one of kinds after it. A stream that -// cannot be read answers a wait that sees nothing, so the act is answered "started". -func (h streamHear) await(kinds []string) func(bound time.Duration) (map[string]any, bool) { - raw, err := h.bus(buscalls.StreamFill, buscalls.StreamRequest{Stream: EventsStream}) - var fill buscalls.StreamFilled - if err == nil { - err = json.Unmarshal(raw, &fill) - } - if err != nil { - return func(time.Duration) (map[string]any, bool) { return nil, false } - } - from := fill.Last + 1 - return func(bound time.Duration) (map[string]any, bool) { - deadline := h.now().Add(bound) - for { - raw, err := h.bus(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: EventsStream, From: from, Limit: 500}) - if err == nil { - var page buscalls.StreamPage - if json.Unmarshal(raw, &page) == nil { - for _, msg := range page.Messages { - from = msg.Sequence + 1 - for _, k := range kinds { - if msg.Subject == k || strings.HasSuffix(msg.Subject, "."+k) { - var body map[string]any - _ = json.Unmarshal(msg.Body, &body) - return body, true - } - } - } - if page.Next != 0 { - from = page.Next - continue - } - } - } - if !h.now().Before(deadline) { - return nil, false - } - h.sleep(HandPoll) - } - } -} - // busLedger is the ledger `snapshots` through the tool runner. type busLedger struct{} diff --git a/modules/zfs/cmd/zfs-tools/bus_stream_test.go b/modules/zfs/cmd/zfs-tools/bus_stream_test.go index 983b348ec..9f97f3ae3 100644 --- a/modules/zfs/cmd/zfs-tools/bus_stream_test.go +++ b/modules/zfs/cmd/zfs-tools/bus_stream_test.go @@ -4,93 +4,10 @@ import ( "encoding/json" "errors" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/buscalls" ) -// fakeStream answers a bundle's calls on its events stream from a list of messages and a clock the wait advances. -type fakeStream struct { - msgs []buscalls.StreamMessage - asked []string - reads []uint64 - now time.Time - fillErr error - // at maps a read count to messages added before that read (an event that comes while the tool waits). - arrive map[int]buscalls.StreamMessage -} - -func (f *fakeStream) ask(method string, params any) (json.RawMessage, error) { - f.asked = append(f.asked, method) - switch method { - case buscalls.StreamFill: - if f.fillErr != nil { - return nil, f.fillErr - } - last := uint64(0) - if n := len(f.msgs); n > 0 { - last = f.msgs[n-1].Sequence - } - return json.Marshal(buscalls.StreamFilled{First: 1, Last: last}) - case buscalls.StreamRead: - r := params.(buscalls.StreamReadRequest) - f.reads = append(f.reads, r.From) - if m, ok := f.arrive[len(f.reads)]; ok { - m.Sequence = uint64(len(f.msgs) + 1) - f.msgs = append(f.msgs, m) - } - page := buscalls.StreamPage{Messages: []buscalls.StreamMessage{}} - for _, m := range f.msgs { - if m.Sequence >= r.From { - page.Messages = append(page.Messages, m) - } - } - return json.Marshal(page) - case buscalls.StreamPublish: - return json.Marshal(buscalls.StreamPublished{Sequence: 9}) - } - return nil, errors.New("unexpected " + method) -} - -func (f *fakeStream) hear() streamHear { - return streamHear{bus: f.ask, now: func() time.Time { return f.now }, sleep: func(d time.Duration) { f.now = f.now.Add(d) }} -} - -// A-12, A-42: a hand act's wait marks the events stream's last sequence before the act starts, then reads from the next -// one: an event before the mark is not the act's, one after it with an end kind is, and with none in 30 s the wait gives up. -func TestAHandWaitHearsTheEventsStreamAfterItsMark(t *testing.T) { - old := buscalls.StreamMessage{Sequence: 1, Subject: "mesh.bus.zfs.events.snapshot.taken.person", Body: json.RawMessage(`{"old":true}`)} - other := buscalls.StreamMessage{Subject: "mesh.bus.zfs.events.config.refused", Body: json.RawMessage(`{}`)} - end := buscalls.StreamMessage{Subject: "mesh.bus.zfs.events.snapshot.taken.person", Body: json.RawMessage(`{"snapshot":"tank@x"}`)} - - f := &fakeStream{msgs: []buscalls.StreamMessage{old}, now: time.Unix(0, 0), arrive: map[int]buscalls.StreamMessage{2: other, 3: end}} - wait := f.hear().await([]string{EventTakenPerson}) - if len(f.asked) != 1 || f.asked[0] != buscalls.StreamFill { - t.Fatalf("the mark was not read first: %v", f.asked) - } - ev, ok := wait(HandWait) - if !ok || ev["snapshot"] != "tank@x" { - t.Fatalf("heard %v %v; the old event or none", ev, ok) - } - if f.reads[0] != 2 { - t.Errorf("read from %d, want the sequence after the mark, 2", f.reads[0]) - } - - g := &fakeStream{msgs: []buscalls.StreamMessage{old}, now: time.Unix(0, 0)} - start := g.now - if ev, ok := g.hear().await([]string{EventTakenPerson})(HandWait); ok || ev != nil { - t.Fatalf("heard %v with nothing new on the stream", ev) - } - if g.now.Sub(start) < HandWait { - t.Errorf("gave up after %s, before the 30 s", g.now.Sub(start)) - } - - h := &fakeStream{fillErr: errors.New("no stream"), now: time.Unix(0, 0)} - if _, ok := h.hear().await([]string{EventTakenPerson})(HandWait); ok { - t.Error("heard an event from a stream that could not be read") - } -} - // A-42: the tool says an event with mesh/stream.publish on the stream `events`, the kind as the event and a message id. func TestAToolSaysItsEventWithStreamPublish(t *testing.T) { var got buscalls.StreamPublishRequest diff --git a/modules/zfs/cmd/zfs-tools/hand.go b/modules/zfs/cmd/zfs-tools/hand.go index 6d30d4c83..38ba54690 100644 --- a/modules/zfs/cmd/zfs-tools/hand.go +++ b/modules/zfs/cmd/zfs-tools/hand.go @@ -4,19 +4,15 @@ import ( "context" "fmt" "strings" - "time" ) // The hand acts (design 42 zfs-owns-zfs §23, §24; A-12, A-17, A-18). // // A hand act writes its act into the state `schedule` under its own key {by, why, at} and starts the unit of // the scheduled run through `/node-service-manager.start` with {scope, unit}; the run reads the key, -// acts under the same lock and guards, and deletes it. The tool waits at most HandWait for the act's event; past -// it, it answers "started", naming the act and that `zfs_module_events` reads its outcome, and never "done" for -// what it did not see end. The `why` is kept in the key. - -// HandWait is how long a hand act's tool waits for the act's event (A-12). -const HandWait = 30 * time.Second +// acts under the same lock and guards, and deletes it. The tool does not wait (A-43): once the key is written and +// the unit started it answers "started" at once, naming the act and that `zfs_module_events` reads its outcome, and +// never "done" for what it did not see end. The `why` is kept in the key. // The acts, as the keys of the state `schedule`. const ( @@ -25,12 +21,8 @@ const ( ActApply = "apply" ) -// handEnds are the events that end each act. -var handEnds = map[string][]string{ - ActSnapshots: {EventTakenPerson, EventSkippedPerson, EventRunSkipped}, - ActPrune: {EventRemovedPerson, EventSkippedPerson, EventRunSkipped}, - ActApply: {EventApplied, EventRefused, EventWaiting}, -} +// handActs are the acts of the run a person may start. +var handActs = map[string]bool{ActSnapshots: true, ActPrune: true, ActApply: true} // StartArgs are what `node-service-manager.start` is called with to run the scheduled run now. func StartArgs() map[string]any { return map[string]any{"scope": "system", "unit": RunService} } @@ -38,10 +30,9 @@ func StartArgs() map[string]any { return map[string]any{"scope": "system", "unit // HandAnswer is what a hand act's tool answers. type HandAnswer struct { Act string `json:"act"` - // Outcome is "seen" (the act's event came) or "started" (it was started, its end not seen). - Outcome string `json:"outcome"` - Event map[string]any `json:"event,omitempty"` - Said string `json:"said"` + // Outcome is "started": the act was started and its end is read from zfs_module_events (A-43). + Outcome string `json:"outcome"` + Said string `json:"said"` } func whyOf(why string) error { @@ -57,7 +48,7 @@ func whyOf(why string) error { // HandAct starts one act of the scheduled run by hand. func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, error) { a := HandAnswer{Act: act} - if _, ok := handEnds[act]; !ok { + if !handActs[act] { return a, fmt.Errorf("%q is not an act of the run: snapshots, prune or apply", act) } if err := whyOf(why); err != nil { @@ -66,10 +57,6 @@ func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, if m.States == nil || m.Call == nil { return a, fmt.Errorf("this process reaches neither the module's state nor the service manager, so %s was not started", act) } - var wait func(time.Duration) (map[string]any, bool) - if m.Await != nil { - wait = m.Await(handEnds[act]) // marks the stream's last sequence before the act starts - } if err := m.States.Put(StateSchedule, act, map[string]any{"by": by, "why": why, "at": m.Now().UTC()}); err != nil { return a, fmt.Errorf("the act %s could not be written to the state %s: %v", act, StateSchedule, err) } @@ -77,15 +64,8 @@ func (m *Machine) HandAct(ctx context.Context, act, by, why string) (HandAnswer, _ = m.States.Delete(StateSchedule, act) return a, fmt.Errorf("the service manager did not start %s: %v", RunService, err) } - if wait != nil { - if ev, ok := wait(HandWait); ok { - a.Outcome, a.Event = "seen", ev - a.Said = fmt.Sprintf("The act %s ran: %v.", act, ev) - return a, nil - } - } a.Outcome = "started" - a.Said = fmt.Sprintf("Started the act %s (%s) and did not see it end within %s: zfs_module_events reads its outcome.", act, RunService, HandWait) + a.Said = fmt.Sprintf("Started the act %s (%s) and did not wait for it to end: zfs_module_events reads its outcome.", act, RunService) return a, nil } diff --git a/modules/zfs/cmd/zfs-tools/hand_test.go b/modules/zfs/cmd/zfs-tools/hand_test.go index 39bf1344e..a9076b71b 100644 --- a/modules/zfs/cmd/zfs-tools/hand_test.go +++ b/modules/zfs/cmd/zfs-tools/hand_test.go @@ -6,26 +6,22 @@ import ( "errors" "strings" "testing" - "time" ) // §23, §24, A-12, A-17, A-18: a hand act writes its key and starts the run's unit with {scope, unit}; the tool -// answers from the act's event when it comes within the wait, else "started", never "done"; a failed start +// answers "started" at once and never "done" (A-43); a failed start // leaves no key; pause writes `paused`, resume deletes it. -func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { +func TestAHandActWritesItsKeyStartsTheRunAndAnswersStartedAtOnce(t *testing.T) { cases := []struct { name, act, why string - event map[string]any callErr error outcome string err string keyLeft bool - waited []string }{ - {name: "event seen", act: ActSnapshots, why: "before the move", event: map[string]any{"snapshot": "tank@mesh-hourly"}, outcome: "seen", keyLeft: true, - waited: handEnds[ActSnapshots]}, - {name: "event not seen in time", act: ActPrune, why: "free space", outcome: "started", keyLeft: true, waited: handEnds[ActPrune]}, - {name: "apply", act: ActApply, why: "now", event: map[string]any{"key": "hostid"}, outcome: "seen", keyLeft: true, waited: handEnds[ActApply]}, + {name: "snapshots", act: ActSnapshots, why: "before the move", outcome: "started", keyLeft: true}, + {name: "prune", act: ActPrune, why: "free space", outcome: "started", keyLeft: true}, + {name: "apply", act: ActApply, why: "now", outcome: "started", keyLeft: true}, {name: "start refused", act: ActSnapshots, why: "x", callErr: errors.New("refused"), err: "did not start"}, {name: "no why", act: ActSnapshots, why: "", err: "say why"}, {name: "unknown act", act: "destroy", why: "x", err: "not an act"}, @@ -37,17 +33,7 @@ func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { m.States, m.Node = st, "ace" var key string var body any - var marked, started bool - m.Call = func(k string, b any) (json.RawMessage, error) { key, body, started = k, b, true; return nil, c.callErr } - var bound time.Duration - var kinds []string - m.Await = func(k []string) func(time.Duration) (map[string]any, bool) { - kinds, marked = k, !started - return func(d time.Duration) (map[string]any, bool) { - bound = d - return c.event, c.event != nil - } - } + m.Call = func(k string, b any) (json.RawMessage, error) { key, body = k, b; return nil, c.callErr } a, err := m.HandAct(context.Background(), c.act, "caller@node", c.why) if c.err != "" { if err == nil || !strings.Contains(err.Error(), c.err) { @@ -65,11 +51,8 @@ func TestAHandActWritesItsKeyStartsTheRunAndAnswersFromItsEvent(t *testing.T) { t.Errorf("%s: called %s with %v", c.name, key, body) } } - if c.err == "" && !marked { - t.Errorf("%s: the stream was not marked before the act started", c.name) - } - if c.err == "" && (bound != 30*time.Second || strings.Join(kinds, ",") != strings.Join(c.waited, ",")) { - t.Errorf("%s: waited %v for %v", c.name, bound, kinds) + if c.outcome == "started" && !strings.Contains(a.Said, "zfs_module_events") { + t.Errorf("%s: did not say where the outcome is read: %s", c.name, a.Said) } if c.outcome == "started" && strings.Contains(a.Said, "done") { t.Errorf("%s: said done for what it did not see: %s", c.name, a.Said) diff --git a/modules/zfs/cmd/zfs-tools/machine.go b/modules/zfs/cmd/zfs-tools/machine.go index fd2e6e0c5..a1be9965b 100644 --- a/modules/zfs/cmd/zfs-tools/machine.go +++ b/modules/zfs/cmd/zfs-tools/machine.go @@ -125,9 +125,6 @@ type Machine struct { Call func(key string, body any) (json.RawMessage, error) // Node is this machine's name, for a seat verb of its own. Node string - // Await marks the module's events stream before an act starts and answers the wait for an event of the module whose - // kind is one of kinds, after the mark: at most the bound, and the event; false when none came (A-12, A-42). - Await func(kinds []string) func(bound time.Duration) (map[string]any, bool) // Findings writes the module's own state `findings` as the seat mesh-conditions' layout says (A-32). Findings views.FindingWriter // Warnings are what the apply saw that is no failure (a change the bus did not take, a parameter awaiting diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index bfa579340..c9e44224a 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -16,7 +16,6 @@ import ( "os" "os/signal" "syscall" - "time" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -73,7 +72,7 @@ func main() { } m := Here() m.Emit = toolEmit(stdio.BusState) - m.States, m.Call, m.Ledger, m.Await, m.Log = toolStates{}, stdio.Ask, busLedger{}, streamHear{bus: stdio.BusState, now: time.Now, sleep: time.Sleep}.await, busStream{} + m.States, m.Call, m.Ledger, m.Log = toolStates{}, stdio.Ask, busLedger{}, busStream{} // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): zfs. if err := stdio.Serve("", Tools(m)); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/modules/zfs/cmd/zfs-tools/tools.go b/modules/zfs/cmd/zfs-tools/tools.go index 6a0adda26..02cd45626 100644 --- a/modules/zfs/cmd/zfs-tools/tools.go +++ b/modules/zfs/cmd/zfs-tools/tools.go @@ -62,9 +62,9 @@ func Tools(m *Machine) []stdio.Tool { return ans, nil }, }, - handTool(m, "zfs_snapshots_run_now", ActSnapshots, "Take the scheduled ZFS snapshots now, as the scheduled run does (a person's act with why): writes the act into the state `schedule` and starts zfs-mesh-run.service through the service manager; the run takes it under the same lock and guards. Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), - handTool(m, "zfs_snapshots_prune_now", ActPrune, "Remove the scheduled ZFS snapshots past their keep count now, as the scheduled run does, only those its own ledger holds (a person's act with why). Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), - handTool(m, "zfs_config_apply", ActApply, "Converge the node to its config file now, as the apply does (a person's act with why): the same lock and guards as the apply. Waits up to 30 seconds for the act's event, else answers started and says zfs_module_events reads its outcome. (a)"), + handTool(m, "zfs_snapshots_run_now", ActSnapshots, "Take the scheduled ZFS snapshots now, as the scheduled run does (a person's act with why): writes the act into the state `schedule` and starts zfs-mesh-run.service through the service manager; the run takes it under the same lock and guards. Does not wait: answers started at once and says zfs_module_events reads its outcome. (a)"), + handTool(m, "zfs_snapshots_prune_now", ActPrune, "Remove the scheduled ZFS snapshots past their keep count now, as the scheduled run does, only those its own ledger holds (a person's act with why). Does not wait: answers started at once and says zfs_module_events reads its outcome. (a)"), + handTool(m, "zfs_config_apply", ActApply, "Converge the node to its config file now, as the apply does (a person's act with why): the same lock and guards as the apply. Does not wait: answers started at once and says zfs_module_events reads its outcome. (a)"), { Name: "zfs_snapshots_pause", Description: "Pause the scheduled ZFS snapshots: writes `paused` {by, why, at} in the state `schedule`; the run then takes and removes nothing until zfs_snapshots_resume. (a)", diff --git a/modules/zfs/module.json b/modules/zfs/module.json index e4c93d234..59f678c73 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -2,8 +2,11 @@ "module": "zfs", "version": "1", "capabilities": [ + "bus-state-calls", + "ledger-calls", "package-manager", - "service-manager" + "service-manager", + "stream-publish" ], "invokes": [ "seat:node-service-manager.start" From d15502f3650dd7021eef600732403f8e8845fdd8 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 02:01:06 +0100 Subject: [PATCH 29/35] zfs: declare health on the four running services, README says hand acts answer started MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §13, §24, lead answers A-45, A-43 --- modules/zfs/README.md | 2 +- modules/zfs/module.json | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/modules/zfs/README.md b/modules/zfs/README.md index 97609eb82..144e808d7 100644 --- a/modules/zfs/README.md +++ b/modules/zfs/README.md @@ -92,7 +92,7 @@ runs out of time is ended with everything it started. | `zfs_config` | every value with its config value, what runs, how it stands, the package's files, and what the module says before it leaves | | `zfs_config_apply` | converge now, as the apply does | | `zfs_snapshots_schedule` | the keep counts, next moments, what the next run would remove, and the ledger | -| `zfs_snapshots_run_now`, `zfs_snapshots_prune_now`, `zfs_snapshots_pause`, `zfs_snapshots_resume` | a person's acts on the scheduled ZFS snapshots; the hand acts wait up to 30 seconds for their event, else answer "started" | +| `zfs_snapshots_run_now`, `zfs_snapshots_prune_now`, `zfs_snapshots_pause`, `zfs_snapshots_resume` | a person's acts on the scheduled ZFS snapshots; the hand acts answer "started" at once and `zfs_module_events` reads the outcome | | `zfs_module_events` | the module's own events, newest first (`zfs_events` reads ZFS's) | | `zfs_snapshots_check` | the ledger against what ZFS holds; changes nothing | | `zfs_findings` | the follower's findings | diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 59f678c73..f0aa57566 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -479,21 +479,30 @@ "type": "service", "unit": "zfs-import-cache.service", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "mount", "type": "service", "unit": "zfs-mount.service", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "target", "type": "service", "unit": "zfs.target", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "import-scan", @@ -530,7 +539,10 @@ "zed-handler-all-syslog", "zed-handler-scrub-after-resilver", "zed-drop-in" - ] + ], + "health": { + "kind": "unit" + } }, { "id": "zed-off", From aa0a472b29242d7198645c47c40b27e7248eab90 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:12:09 +0100 Subject: [PATCH 30/35] mesh-conditions: the keeper reads zfs's findings (A-32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group's check failed TestEveryControllerKindHasItsHome: the reader zfs's findings are not read by the keeper. Builds: design 42 zfs-owns-zfs §27, A-32 --- modules/mesh-conditions/cmd/mesh-conditions/keeper.go | 2 ++ modules/mesh-conditions/module.json | 1 + 2 files changed, 3 insertions(+) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 006aac7aa..294ee708e 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -67,6 +67,8 @@ var Observers = []string{watchmanModule, selfCheckModule, healersModule, assigne builder, indexer, "mesh-delivery-checker", "mesh-delivery-teller", "mesh-delivery-view", enrolment, // The tool runner's and claude-code's own first views (views.UnansweredFinding in their findings, ADR 0615 §6). "node-tools", "claude-code", + // The zfs module's: its findings state (A-32). + "zfs", // The node-engine's: a node that has read no seat list five minutes after its start, seat-list-unread (ADR 0636 §1; // issue 1966: written on ace and raised by nobody). buscompose.EngineModule} diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index f6803f865..753c14f9a 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -34,6 +34,7 @@ "mesh-enrolment.findings", "node-tools.findings", "claude-code.findings", + "zfs.findings", "mesh-host.findings", "mesh-healers.attempts", "node-engine.have" From 8ae4fabfbe2972af334a7c6e197bb45914126d8e Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:12:40 +0100 Subject: [PATCH 31/35] zfs: the keeper tests make their socket directory in the test's own temporary directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group's check failed TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown and TestAKeeperThatDoesNotAnswerIsWarnedOnce: stat /var/tmp: no such file or directory. Builds: design 42 zfs-owns-zfs §18, A-37 --- modules/zfs/cmd/zfs-tools/keeper_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/keeper_test.go b/modules/zfs/cmd/zfs-tools/keeper_test.go index c27ee3e72..bd4bd1b44 100644 --- a/modules/zfs/cmd/zfs-tools/keeper_test.go +++ b/modules/zfs/cmd/zfs-tools/keeper_test.go @@ -73,7 +73,7 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. k := &Keeper{Emit: func(string, any) error { return nil }, States: st, Call: func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }, PeerOf: func(net.Conn) (int, error) { return uid, nil }} - dir, err := os.MkdirTemp("/var/tmp", "zk") + dir, err := os.MkdirTemp("", "zk") if err != nil { t.Fatal(err) } @@ -148,7 +148,7 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. // A-37: a root process whose keeper does not answer acts as with the bus down and says so, once, in a warning, for the // hand acts, the forgetting of applied keys and the pause alike. func TestAKeeperThatDoesNotAnswerIsWarnedOnce(t *testing.T) { - dir, err := os.MkdirTemp("/var/tmp", "keeper-down") + dir, err := os.MkdirTemp("", "keeper-down") if err != nil { t.Fatal(err) } From 064ebc9b849fd19975b80ef267ef0021320dc910 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:13:39 +0100 Subject: [PATCH 32/35] zfs: the zed-functions file names its notification services on purpose, with its reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group's check failed TestRulesBus/NoCatalogueManifestNamesAnInstallation: 11 values name an installation (the vendor sites in the package's zed-functions.sh, resources[13]). Builds: design 42 zfs-owns-zfs §14, A-2 --- modules/zfs/module.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/zfs/module.json b/modules/zfs/module.json index f0aa57566..ef2fb0e1a 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -432,7 +432,16 @@ "template": "files/zed-functions.sh", "if-config": [ "zed=on" - ] + ], + "names-on-purpose": { + "api.pushbullet.com": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "api.pushover.net": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "api.slack.com": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "docs.pushbullet.com": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "gotify.net": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "pushover.net": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it", + "www.pushbullet.com": "the distribution's own zed-functions.sh, kept as the package ships it (A-2): its notifier functions name the notification services' sites in their comments and urls, and none is used unless the operator's zed.rc enables it" + } }, { "id": "zed-handler-all-syslog", From afa825f902a2541e34f70b5aa19f2cd7f75bc215 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:15:19 +0100 Subject: [PATCH 33/35] mesh-hand-acts: the whole test dates its warrant from now, inside the 14-day window Issue 1979, blocking: a date-bound test. The warrant dated 2026-10-29 fell out of the listing's 14 days on 2026-11-13, failing every mesh-catalog check. Builds: design 42 zfs-owns-zfs (the group's check) --- .../cmd/mesh-hand-acts/whole_integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go index 555208eb6..2b167edfb 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -392,9 +392,11 @@ func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { } // A warrant the router recorded, asked twice: one act. asks, _ := js.KeyValue(ctx, "messenger_asks") + // Dated from now: the listing holds the last 14 days, so a fixed date goes out of the window (issue 1979). + warrantAt := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code", "outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram", - "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil { + "at":"`+warrantAt+`","ask-digest":"sha256:x"}}`)); err != nil { t.Fatal(err) } for _, caller := range []string{"ace.claude-code", "g14.claude-code"} { From d1a15ced6c5c3d51d7011b9c9665f1810474df97 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:15:30 +0100 Subject: [PATCH 34/35] zfs: the keeper tests' socket directory is the shortest that works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the test's temporary directory, or /tmp when the socket path would pass 100 characters, so a long TMPDIR cannot fail the bind. Builds: design 42 zfs-owns-zfs §18, A-37 --- modules/zfs/cmd/zfs-tools/keeper_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/keeper_test.go b/modules/zfs/cmd/zfs-tools/keeper_test.go index bd4bd1b44..d1c790a0e 100644 --- a/modules/zfs/cmd/zfs-tools/keeper_test.go +++ b/modules/zfs/cmd/zfs-tools/keeper_test.go @@ -65,6 +65,17 @@ func TestTheKeeperAnswersOnlyItsClosedListOfCalls(t *testing.T) { } } +// shortTempDir is a directory in the test's temporary directory, or in /tmp when a socket in it (`k/k.sock`) would pass 100 +// characters, the most a unix socket path takes, so a long TMPDIR cannot fail the bind. +func shortTempDir(prefix string) (string, error) { + dir, err := os.MkdirTemp("", prefix) + if err != nil || len(filepath.Join(dir, "k", "k.sock")) <= 100 { + return dir, err + } + os.RemoveAll(dir) + return os.MkdirTemp("/tmp", prefix) +} + // A-37: only root opens the socket; a root process says and reads through it as through a bus of its own, and with no // keeper answering it acts as with the bus down, with a warning (A-8). func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing.T) { @@ -73,7 +84,7 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. k := &Keeper{Emit: func(string, any) error { return nil }, States: st, Call: func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }, PeerOf: func(net.Conn) (int, error) { return uid, nil }} - dir, err := os.MkdirTemp("", "zk") + dir, err := shortTempDir("zk") if err != nil { t.Fatal(err) } @@ -148,7 +159,7 @@ func TestOnlyRootOpensTheKeepersSocketAndANeedlessKeeperIsTheBusDown(t *testing. // A-37: a root process whose keeper does not answer acts as with the bus down and says so, once, in a warning, for the // hand acts, the forgetting of applied keys and the pause alike. func TestAKeeperThatDoesNotAnswerIsWarnedOnce(t *testing.T) { - dir, err := os.MkdirTemp("", "keeper-down") + dir, err := shortTempDir("kd") if err != nil { t.Fatal(err) } From 0041b69d173611afb395e7c0f6891b8d89ffe64f Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 03:27:38 +0100 Subject: [PATCH 35/35] zfs: findings is a state of the module's state list, not of requires, so the conditions keeper reads it The validator refused zfs and mesh-conditions: mesh-conditions reads zfs.findings, which zfs had moved to bus state in requires, where the new form has no readers (ADR 0596 section 2). The keeper process's bus.state keeps naming findings. The manifest test holds it. Builds: design 42 zfs-owns-zfs A-46 (A-32, 26, 27) --- modules/zfs/cmd/zfs-tools/manifest_test.go | 8 +++++--- modules/zfs/module.json | 8 +++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index cac180233..9572dd4db 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -480,13 +480,15 @@ func TestTheManifestsBusStateKeeperAndTools(t *testing.T) { if ev["storage"] != "stream" || ev["template"] != "events" || opts["max-age-days"] != float64(30) || strings.Join(kinds, ",") != strings.Join(EventKinds, ",") { t.Errorf("the events stream: %v", ev) } - for _, n := range []string{"applied", "schedule", "findings"} { + for _, n := range []string{"applied", "schedule"} { if e := reqs[n]; e == nil || e["storage"] != "key-value" || e["template"] != "state" { t.Errorf("the state %s: %v", n, e) } } - if reqs["findings"]["options"] != nil { - t.Errorf("findings declares no stream of its own (A-32): %v", reqs["findings"]) + // A-46: findings is a state of the module's `state` list, as every observer's, and not asked for in requires, where the + // conditions keeper would not read it. + if reqs["findings"] != nil || fmt.Sprint(tree["state"]) != "[findings]" { + t.Errorf("findings is a state of the state list, not of requires (A-46): %v, %v", tree["state"], reqs["findings"]) } l := reqs[LedgerName] lo, _ := l["options"].(map[string]any) diff --git a/modules/zfs/module.json b/modules/zfs/module.json index ef2fb0e1a..ef1a00543 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -17,6 +17,9 @@ "owner": "zfs-keeper" } }, + "state": [ + "findings" + ], "requires": [ { "name": "events", @@ -49,11 +52,6 @@ "storage": "key-value", "template": "state" }, - { - "name": "findings", - "storage": "key-value", - "template": "state" - }, { "name": "snapshots", "storage": "ledger",