package main import ( "context" "encoding/json" "fmt" "regexp" "strings" "time" ) // The run's ZFS snapshots: taking them (design 42 zfs-owns-zfs §21, A-20, A-30, A-31). // // Each kind whose keep count is above 0 is due at the first run of the hour (hourly), the first run after // midnight (daily), on Monday (weekly) and on the first of the month (monthly), in the node's time zone, // against the ledger's last entry of that kind. For every filesystem of the pool the run takes `zfs snapshot -- // @mesh--`, never recursive, sets `mesh:schedule=` on it, and records `taken` // through the module's own tool `zfs_snapshots_record` (a process cannot write a ledger, A-20). It reads the // ledger through `zfs_snapshots_schedule` (A-31). It takes none it cannot record in the same run: with the bus // down or the ledger full it takes none and says `snapshot.skipped.schedule`, and none while the pool is more // than 80 % full. // FullPercent is the capacity above which no scheduled ZFS snapshot is taken. const FullPercent = 80 // The tools the run calls on its own module (A-20, A-31). const ( ToolRecord = "zfs.zfs_snapshots_record" ToolSchedule = "zfs.zfs_snapshots_schedule" ) // StampForm is the UTC stamp in a scheduled snapshot's name. const StampForm = "20060102T1504Z" var meshSnapshot = regexp.MustCompile(`^mesh-(hourly|daily|weekly|monthly)-([0-9]{8}T[0-9]{4}Z)$`) // SnapshotSetArgv sets a user property on a snapshot; only mesh:schedule is ever set by the module. func SnapshotSetArgv(snapshot, kind string) []string { return []string{"set", "mesh:schedule=" + kind, "--", snapshot} } // LedgerEntry is one instance of the ledger `snapshots` (A-30): the ZFS snapshot the run took. type LedgerEntry struct { Dataset string `json:"dataset"` Name string `json:"name"` Kind string `json:"kind"` At string `json:"at"` } // LedgerInstance is the instance path of a scheduled snapshot: `snapshot/--`, the // dataset's `/` written `~`. func LedgerInstance(kind, stamp, dataset string) string { return "snapshot/" + kind + "-" + stamp + "-" + strings.ReplaceAll(dataset, "/", "~") } // InstanceOf is the instance of a record's snapshot, refusing a name that is not a scheduled snapshot's. func InstanceOf(dataset, name string) (string, error) { m := meshSnapshot.FindStringSubmatch(name) if m == nil { return "", fmt.Errorf("%q is not a scheduled snapshot's name (mesh--): the ledger holds only the run's", name) } if dataset == "" || strings.ContainsAny(dataset, "@~ ") { return "", fmt.Errorf("%q is not a dataset's name", dataset) } return LedgerInstance(m[1], m[2], dataset), nil } // ledgerEntries reads the ledger through the module's tool zfs_snapshots_schedule. func (m *Machine) ledgerEntries() ([]LedgerEntry, error) { if m.Call == nil { return nil, fmt.Errorf("this process reaches no tool, so the ledger cannot be read") } raw, err := m.Call(ToolSchedule, map[string]any{}) if err != nil { return nil, err } var ans struct { Ledger []LedgerEntry `json:"ledger"` } if err := json.Unmarshal(raw, &ans); err != nil { return nil, fmt.Errorf("zfs_snapshots_schedule answered what is no ledger: %v", err) } return ans.Ledger, nil } // record tells the ledger of a taken or removed snapshot through zfs_snapshots_record. func (m *Machine) recordLedger(op string, e LedgerEntry) error { if m.Call == nil { return fmt.Errorf("this process reaches no tool, so %s was not recorded", e.Name) } _, err := m.Call(ToolRecord, map[string]any{"operation": op, "dataset": e.Dataset, "name": e.Name, "kind": e.Kind, "at": e.At}) return err } // keepsOf are the kinds with a keep count above 0, in the order they are said. func keepsOf(k *Keeps) []string { var out []string for _, kind := range keepKinds { if k.Of(kind) > 0 { out = append(out, kind) } } return out } // dueKinds are the kinds due now: the calendar's moment of the kind, and no entry of the kind since its period began. func dueKinds(k *Keeps, now time.Time, entries []LedgerEntry) []string { day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) var due []string for _, kind := range keepsOf(k) { start := day switch kind { case "hourly": start = now.Truncate(time.Hour) case "weekly": if now.Weekday() != time.Monday { continue } case "monthly": if now.Day() != 1 { continue } } var last time.Time for _, e := range entries { if at, err := time.Parse(time.RFC3339, e.At); err == nil && e.Kind == kind && at.After(last) { last = at } } if last.Before(start) { due = append(due, kind) } } return due } // ledgerCap is the ledger's bound: the keep counts' sum times two instances per dataset (§26). func ledgerCap(k *Keeps, datasets int) int { return (k.Hourly + k.Daily + k.Weekly + k.Monthly) * 2 * datasets } func (m *Machine) skippedSnapshots(person bool, pool string, capacity int, reason string) { kind := EventSkippedSched if person { kind = EventSkippedPerson } var failed []error if err := m.say(kind, map[string]any{"pool": pool, "capacity": capacity, "reason": reason}); err != nil { m.failure(&failed, kind, err) } } // filesystems are the pool's filesystems, never a volume. func (m *Machine) filesystems(ctx context.Context, pool string) ([]string, int, error) { ds, err := m.Datasets(ctx) if err != nil { return nil, 0, err } var out []string for _, d := range ds { if d.Type == "filesystem" && (d.Name == pool || strings.HasPrefix(d.Name, pool+"/")) { out = append(out, d.Name) } } capacity := 0 ps, err := m.Pools(ctx) if err != nil { return nil, 0, err } for _, p := range ps { if p.Name == pool { capacity = p.Capacity } } return out, capacity, nil } // TakeScheduled takes the due ZFS snapshots (every kind with a keep count when force: a person's act, by // them). It answers what it did, and a failed command as the error. func (m *Machine) TakeScheduled(ctx context.Context, c *Config, force bool, by string) ([]string, error) { if c.Snapshots == nil || c.Snapshots.None || c.Pool == "" { return nil, nil } person := force taken := EventTakenSchedule if person { taken = EventTakenPerson } else { by = "schedule" } fs, capacity, err := m.filesystems(ctx, c.Pool) if err != nil { return nil, err } if m.Paused() { m.skippedSnapshots(person, c.Pool, capacity, "paused") return []string{"snapshots paused"}, nil } entries, err := m.ledgerEntries() if err != nil { m.skippedSnapshots(person, c.Pool, capacity, "ledger-unavailable: "+err.Error()) return []string{"no snapshot taken: the ledger cannot be read"}, nil } due := keepsOf(c.Snapshots) if !force { due = dueKinds(c.Snapshots, m.Now(), entries) } if len(due) == 0 || len(fs) == 0 { return nil, nil } if capacity > FullPercent { m.skippedSnapshots(person, c.Pool, capacity, "full") return []string{fmt.Sprintf("no snapshot taken: the pool is %d %% full", capacity)}, nil } if len(entries)+len(due)*len(fs) > ledgerCap(c.Snapshots, len(fs)) { m.skippedSnapshots(person, c.Pool, capacity, "ledger-full") return []string{"no snapshot taken: the ledger is full"}, nil } now := m.Now().UTC() var said []string var failed []error for _, kind := range due { for _, ds := range fs { name := fmt.Sprintf("%s%s-%s", MeshPrefix, kind, now.Format(StampForm)) full := ds + "@" + name if _, err := m.AsRoot(ctx, "zfs", SnapshotArgv(ds, name)...); err != nil { failed = append(failed, fmt.Errorf("%s: %v", full, err)) continue } if _, err := m.AsRoot(ctx, "zfs", SnapshotSetArgv(full, kind)...); err != nil { failed = append(failed, fmt.Errorf("%s was taken, and mesh:schedule could not be set: %v", full, err)) continue } e := LedgerEntry{Dataset: ds, Name: name, Kind: kind, At: now.Format(time.RFC3339)} if err := m.recordLedger("taken", e); err != nil { failed = append(failed, fmt.Errorf("%s was taken, and the ledger did not take it: %v", full, err)) continue } if err := m.say(taken, map[string]any{"snapshot": full, "by": by}); err != nil { m.failure(&failed, taken, err) } said = append(said, "took "+full) } } return said, joinErrors(failed) } // LedgerName is the ledger's local name in the module's `requires`. const LedgerName = "snapshots" // LedgerStore is the ledger `snapshots` as the tools bundle reaches it (A-20, A-30). type LedgerStore interface { // Apply applies an operation to an instance; call is the id of the tool call it serves. Apply(instance, operation string, body any, call string) error // Entries are every instance's state. Entries() ([]LedgerEntry, error) } // Record applies the run's `taken` or `removed` to the ledger, refusing a snapshot not named mesh- (A-20). func (m *Machine) Record(op string, e LedgerEntry, call string) error { if op != "taken" && op != "removed" { return fmt.Errorf("%q is not an operation of the ledger: taken or removed", op) } instance, err := InstanceOf(e.Dataset, e.Name) if err != nil { return err } if m.Ledger == nil { return fmt.Errorf("this process reaches no ledger") } if op == "removed" { return m.Ledger.Apply(instance, op, struct{}{}, call) } return m.Ledger.Apply(instance, op, e, call) }