package main import ( "context" "encoding/json" "fmt" "sort" "strings" "time" ) // The module's own read tools (design 42 zfs-owns-zfs §28, §30): what is owned and how it stands, the // schedule of the scheduled ZFS snapshots with the ledger, the module's events, the ledger against ZFS, and the // follower's findings. None of them changes anything. // configKeys are the config values the module owns, in the order they are said. var configKeys = []string{"pool", "scrub-cadence", "scrub-day", "scrub-time", "arc-max", "arc-min", "module-options", "hostid", "zed", "zed-handlers", "trim-schedule", "trim-day", "trim-time", "cache-devices", "log-devices", "log-mirror", "pool-properties", "dataset-properties", "snapshots"} // appliedKeys are the values the apply converges (the others are the node-engine's files or the run's schedule): // one of them set and with no key in the state `applied` has not been applied yet. var appliedKeys = map[string]bool{"arc-max": true, "arc-min": true, "module-options": true, "hostid": true, "cache-devices": true, "log-devices": true, "log-mirror": true, "pool-properties": true, "dataset-properties": true} // The states a value is shown in (§28). const ( StateNotManaged = "not managed" StateAsSet = "as set" StateNotYet = "not applied yet" StateWaiting = "waiting" StateRefused = "refused" StateFailed = "failed" StateAwaitingLoad = "takes effect at the next load of the kernel module" ) // ConfigValue is one owned config value as zfs_config shows it. type ConfigValue struct { Key string `json:"key"` // Value is what the config file holds, absent when the value is not managed. Value json.RawMessage `json:"value,omitempty"` // Running is what runs on the node, where it is read live; else what the last apply read back. Running string `json:"running,omitempty"` State string `json:"state"` Why string `json:"why,omitempty"` WaitsOn string `json:"waits-on,omitempty"` Since string `json:"since,omitempty"` Note string `json:"note,omitempty"` } // ConfigFileState is one of the package's files the module never writes (§10). type ConfigFileState struct { Path string `json:"path"` Present bool `json:"present"` // Pacnew says a `.pacnew` lies beside it: the package shipped a newer file the module did not take. Pacnew bool `json:"pacnew,omitempty"` } // ConfigAnswer is what zfs_config answers. type ConfigAnswer struct { Values []ConfigValue `json:"values"` Files []ConfigFileState `json:"files"` Leaving []string `json:"leaving"` Said []string `json:"said"` } // packageFiles are the files of §10: the package's, which the module never writes, and ZFS's own caches. var packageFiles = []string{"/etc/default/zfs", "/etc/zfs/zed.d", "/etc/zfs/zed.d/zed.rc", "/etc/zfs/zpool.cache", "/etc/zfs/zfs-list.cache"} // leavingSaid is what the module says before it leaves a node (§30). var leavingSaid = []string{ "When the module leaves, the next load of the kernel module uses ZFS's own defaults: zfs.conf is removed.", "The units zfs-import-cache.service, zfs-mount.service and zfs.target stay running and enabled; they are never stopped or disabled.", "/etc/hostid stays: it is no resource the node-engine removes, and removing it would change the node's hostid, so that the pool's next import could need zpool import -f.", "Pool and dataset properties and devices stay as the last apply set them.", "The distribution's scrub and trim timers are put back as the module found them.", } func (m *Machine) exists(path string) bool { return m.Exists != nil && m.Exists(path) } func rawManaged(raw json.RawMessage) bool { t := strings.TrimSpace(string(raw)) switch t { case "", "null", `""`, "[]", "{}": return false } return true } // live is a kernel parameter read as it runs. func (m *Machine) live(name string) (string, bool) { b, err := m.ReadFile(ParametersDir + name) if err != nil { return "", false } return strings.TrimSpace(string(b)), true } // worse orders the outcomes of a value's keys: the one a person must see first. var worse = map[string]int{OutcomeApplied: 1, OutcomeAwaitingLoad: 2, OutcomeWaiting: 3, OutcomeFailed: 4, OutcomeRefused: 5} // ConfigNow shows every value the module owns: what the config file holds, what runs, how it stands, and the // files it never writes. func (m *Machine) ConfigNow(ctx context.Context) (ConfigAnswer, error) { a := ConfigAnswer{Values: []ConfigValue{}, Files: []ConfigFileState{}, Leaving: leavingSaid, Said: []string{}} text, err := m.ReadFile(ConfigFile) if err != nil { return a, fmt.Errorf("%s could not be read: %v", ConfigFile, err) } raw := map[string]json.RawMessage{} if err := json.Unmarshal(text, &raw); err != nil { return a, fmt.Errorf("%s is not a JSON object: %v", ConfigFile, err) } applied := map[string]AppliedEntry{} if m.States != nil { keys, err := m.States.Keys(StateApplied) if err != nil { return a, fmt.Errorf("the state %s could not be read: %v", StateApplied, err) } for _, k := range keys { if v, ok, _ := m.States.Get(StateApplied, k); ok { var e AppliedEntry if json.Unmarshal(v, &e) == nil { applied[k] = e } } } } for _, key := range configKeys { v := ConfigValue{Key: key, State: StateNotManaged} if !rawManaged(raw[key]) { a.Values = append(a.Values, v) continue } v.Value = raw[key] v.State = StateAsSet if appliedKeys[key] { v.State = StateNotYet } best := 0 var afters []string for _, k := range sortedKeys(applied) { if first, _, _ := strings.Cut(k, "."); first != key { continue } e := applied[k] if e.After != "" { afters = append(afters, k+"="+e.After) } if worse[e.Outcome] > best { best = worse[e.Outcome] v.Why, v.WaitsOn = e.Why, e.WaitsOn v.Since = "" if !e.Since.IsZero() { v.Since = e.Since.Format(time.RFC3339) } switch e.Outcome { case OutcomeApplied: v.State = StateAsSet case OutcomeAwaitingLoad: v.State = StateAwaitingLoad case OutcomeWaiting: v.State = StateWaiting case OutcomeFailed: v.State = StateFailed case OutcomeRefused: v.State = StateRefused } } } v.Running = strings.Join(afters, ", ") switch key { case "arc-max", "arc-min": if n, ok := m.live("zfs_" + strings.ReplaceAll(key, "-", "_")); ok { v.Running = n + " bytes" } case "hostid": if b, err := m.ReadFile(HostIDFile); err == nil { if id, ok := nativeHostID(b); ok { v.Running = fmt.Sprintf("%08x", id) } } v.Note = "a change of the node's ZFS id needs a check of the pool's import at the next boot; the module never reboots" } a.Values = append(a.Values, v) } for _, p := range packageFiles { f := ConfigFileState{Path: p, Present: m.exists(p), Pacnew: m.exists(p + ".pacnew")} a.Files = append(a.Files, f) if f.Pacnew { a.Said = append(a.Said, p+".pacnew lies beside it: the package shipped a newer file the module does not take.") } } a.Said = append([]string{fmt.Sprintf("%d of %d values are set.", countSet(a.Values), len(a.Values))}, a.Said...) return a, nil } func countSet(vs []ConfigValue) int { n := 0 for _, v := range vs { if v.State != StateNotManaged { n++ } } return n } // ScheduleAnswer is what zfs_snapshots_schedule answers. type ScheduleAnswer struct { Pool string `json:"pool,omitempty"` Keeps map[string]int `json:"keeps"` Paused map[string]any `json:"paused,omitempty"` // Next is the moment each kind with a keep count next falls due; the run fires every quarter of an hour, so the // ZFS snapshot is taken at the first run at or after it. Next map[string]string `json:"next"` // WouldRemove are the ledger's entries past their keep count: what the next run removes, if it passes its checks. WouldRemove []LedgerEntry `json:"would-remove"` Ledger []LedgerEntry `json:"ledger"` Said []string `json:"said"` } // nextDue is the first moment after now at which a kind falls due, in now's time zone. func nextDue(kind string, now time.Time) time.Time { day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) switch kind { case "hourly": return now.Truncate(time.Hour).Add(time.Hour) case "weekly": d := day.AddDate(0, 0, 1) for d.Weekday() != time.Monday { d = d.AddDate(0, 0, 1) } return d case "monthly": return time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, now.Location()) } return day.AddDate(0, 0, 1) } // wouldRemove are the entries past their dataset's and kind's keep count; a kind with 0 removes nothing (A-35). func wouldRemove(entries []LedgerEntry, k *Keeps) []LedgerEntry { groups := map[string][]LedgerEntry{} for _, e := range entries { g := e.Dataset + " " + e.Kind groups[g] = append(groups[g], e) } out := []LedgerEntry{} for _, g := range sortedKeys(groups) { list := groups[g] if n := k.Of(list[0].Kind); n > 0 { out = append(out, surplus(list, n)...) } } return out } // SnapshotSchedule shows the schedule of the scheduled ZFS snapshots, its next run, what the next run would remove and // the ledger. The ledger is shown even when the config says nothing of it: the scheduled run reads it from here. func (m *Machine) SnapshotSchedule(ctx context.Context) (ScheduleAnswer, error) { a := ScheduleAnswer{Keeps: map[string]int{}, Next: map[string]string{}, WouldRemove: []LedgerEntry{}, Ledger: []LedgerEntry{}, Said: []string{}} if m.Ledger == nil { return a, fmt.Errorf("this process reaches no ledger") } entries, err := m.ledgerAll(ctx) if err != nil { return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) } if entries != nil { a.Ledger = entries } if m.States != nil { if raw, ok, err := m.States.Get(StateSchedule, KeyPaused); err == nil && ok { _ = json.Unmarshal(raw, &a.Paused) a.Said = append(a.Said, "The schedule is paused: nothing is taken or removed until zfs_snapshots_resume.") } } text, err := m.ReadFile(ConfigFile) if err != nil { a.Said = append(a.Said, ConfigFile+" could not be read: "+err.Error()) return a, nil } c, refused := ParseConfig(text, m.memMiB()) switch { case len(refused) > 0: a.Said = append(a.Said, "The config is refused ("+refused[0].Key+" "+refused[0].Reason+"): the schedule is not known.") case c.Snapshots == nil: a.Said = append(a.Said, "snapshots is not managed: nothing is taken or removed.") case c.Snapshots.None: a.Pool = c.Pool a.Said = append(a.Said, "snapshots is none: nothing is taken, and the scheduled ZFS snapshots that exist are kept.") default: a.Pool = c.Pool for _, kind := range keepKinds { a.Keeps[kind] = c.Snapshots.Of(kind) if a.Keeps[kind] > 0 { a.Next[kind] = nextDue(kind, m.Now()).Format(time.RFC3339) } } a.WouldRemove = wouldRemove(a.Ledger, c.Snapshots) a.Said = append(a.Said, fmt.Sprintf("%d scheduled ZFS snapshots are in the ledger; the next run would remove %d.", len(a.Ledger), len(a.WouldRemove))) } return a, nil } // LoggedEvent is one event of the module's stream. type LoggedEvent struct { Sequence uint64 `json:"sequence"` Kind string `json:"kind"` At time.Time `json:"at"` Body json.RawMessage `json:"body,omitempty"` } // EventLog is the module's own events stream as the tools bundle reaches it (§25). type EventLog interface { // Last are the last n events, oldest first, only those of the kind when one is named. Last(n int, kind string) ([]LoggedEvent, error) } // EventKinds are the module's kinds of event (§25). var EventKinds = []string{EventApplied, EventRefused, EventWaiting, EventTakenPerson, EventTakenSchedule, EventRemovedPerson, EventRemovedSched, EventSkippedPerson, EventSkippedSched, EventRunSkipped, EventScrubRequested} // KindOfSubject is the kind an event's subject ends in. func KindOfSubject(subject string) string { for _, k := range EventKinds { if subject == k || strings.HasSuffix(subject, "."+k) { return k } } return subject } // ModuleEventsAnswer is what zfs_module_events answers. type ModuleEventsAnswer struct { Events []LoggedEvent `json:"events"` ByKind map[string]int `json:"by-kind"` Said []string `json:"said"` } // ModuleEvents are the last n events of the module's stream, newest first, of one kind when it is named. func (m *Machine) ModuleEvents(n int, kind string) (ModuleEventsAnswer, error) { a := ModuleEventsAnswer{Events: []LoggedEvent{}, ByKind: map[string]int{}, Said: []string{}} if kind != "" { known := false for _, k := range EventKinds { known = known || k == kind } if !known { return a, fmt.Errorf("%q is not a kind of this module's events: %s", kind, strings.Join(EventKinds, ", ")) } } if m.Log == nil { return a, fmt.Errorf("this process reaches no events stream") } evs, err := m.Log.Last(n, kind) if err != nil { return a, fmt.Errorf("the events stream could not be read: %v", err) } for i := len(evs) - 1; i >= 0; i-- { a.Events = append(a.Events, evs[i]) a.ByKind[evs[i].Kind]++ } a.Said = append(a.Said, fmt.Sprintf("%d events shown, newest first; the stream keeps 30 days.", len(a.Events))) return a, nil } // CheckAnswer is what zfs_snapshots_check answers. type CheckAnswer struct { // Gone are the ledger's entries whose ZFS snapshot ZFS no longer has. Gone []LedgerEntry `json:"gone"` // Unrecorded are the snapshots named mesh- that ZFS has and the ledger does not. Unrecorded []string `json:"unrecorded"` Said []string `json:"said"` } // CheckSnapshots compares the ledger with what ZFS holds; it changes nothing. func (m *Machine) CheckSnapshots(ctx context.Context) (CheckAnswer, error) { a := CheckAnswer{Gone: []LedgerEntry{}, Unrecorded: []string{}, Said: []string{}} if m.Ledger == nil { return a, fmt.Errorf("this process reaches no ledger") } entries, err := m.ledgerAll(ctx) if err != nil { return a, fmt.Errorf("the ledger %s could not be read: %v", LedgerName, err) } snaps, err := m.Snapshots(ctx) if err != nil { return a, err } have := map[string]bool{} for _, s := range snaps { have[s.Name] = true } recorded := map[string]bool{} for _, e := range entries { full := e.Dataset + "@" + e.Name recorded[full] = true if !have[full] { a.Gone = append(a.Gone, e) } } for _, s := range snaps { if strings.HasPrefix(s.Snap, MeshPrefix) && !recorded[s.Name] { a.Unrecorded = append(a.Unrecorded, s.Name) } } sort.Strings(a.Unrecorded) a.Said = append(a.Said, fmt.Sprintf("%d ledger entries, %d with no ZFS snapshot, %d mesh- snapshots not in the ledger. Nothing was changed.", len(entries), len(a.Gone), len(a.Unrecorded))) return a, nil } // FindingsAnswer is what zfs_findings answers. type FindingsAnswer struct { Findings []FindingKey `json:"findings"` Said []string `json:"said"` } // FindingKey is one key of the follower's state `findings`. type FindingKey struct { Key string `json:"key"` Value json.RawMessage `json:"value"` } // FindingsNow are the follower's findings: one key per pool or device not ONLINE. func (m *Machine) FindingsNow() (FindingsAnswer, error) { a := FindingsAnswer{Findings: []FindingKey{}, Said: []string{}} if m.States == nil { return a, fmt.Errorf("this process reaches no state") } keys, err := m.States.Keys(StateFindings) if err != nil { return a, fmt.Errorf("the state %s could not be read: %v", StateFindings, err) } sort.Strings(keys) for _, k := range keys { if v, ok, err := m.States.Get(StateFindings, k); err == nil && ok { a.Findings = append(a.Findings, FindingKey{Key: k, Value: v}) } } if len(a.Findings) == 0 { a.Said = append(a.Said, "No pool or device is not ONLINE: the follower holds no finding.") } else { a.Said = append(a.Said, fmt.Sprintf("%d findings: a pool or device not ONLINE.", len(a.Findings))) } return a, nil } // ledgerAll reads the ledger's entries for every filesystem of the config's pool (A-36: the ledger is one instance per // dataset and kind, so the datasets name what to read). No pool set, no entries. func (m *Machine) ledgerAll(ctx context.Context) ([]LedgerEntry, error) { text, err := m.ReadFile(ConfigFile) if err != nil { return nil, fmt.Errorf("%s could not be read: %v", ConfigFile, err) } var c struct { Pool string `json:"pool"` } _ = json.Unmarshal(text, &c) if c.Pool == "" { return nil, nil } fs, _, err := m.filesystems(ctx, c.Pool) if err != nil { return nil, err } return m.Ledger.Entries(fs) }