276 lines
9.1 KiB
Go
276 lines
9.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// The run's ZFS snapshots: taking them (design 42 zfs-owns-zfs §21, A-20, A-30, A-31).
|
|
//
|
|
// Each kind whose keep count is above 0 is due at the first run of the hour (hourly), the first run after
|
|
// midnight (daily), on Monday (weekly) and on the first of the month (monthly), in the node's time zone,
|
|
// against the ledger's last entry of that kind. For every filesystem of the pool the run takes `zfs snapshot --
|
|
// <dataset>@mesh-<kind>-<UTC stamp>`, never recursive, sets `mesh:schedule=<kind>` on it, and records `taken`
|
|
// through the module's own tool `zfs_snapshots_record` (a process cannot write a ledger, A-20). It reads the
|
|
// ledger through `zfs_snapshots_schedule` (A-31). It takes none it cannot record in the same run: with the bus
|
|
// down or the ledger full it takes none and says `snapshot.skipped.schedule`, and none while the pool is more
|
|
// than 80 % full.
|
|
|
|
// FullPercent is the capacity above which no scheduled ZFS snapshot is taken.
|
|
const FullPercent = 80
|
|
|
|
// The tools the run calls on its own module (A-20, A-31).
|
|
const (
|
|
ToolRecord = "zfs.zfs_snapshots_record"
|
|
ToolSchedule = "zfs.zfs_snapshots_schedule"
|
|
)
|
|
|
|
// StampForm is the UTC stamp in a scheduled snapshot's name.
|
|
const StampForm = "20060102T1504Z"
|
|
|
|
var meshSnapshot = regexp.MustCompile(`^mesh-(hourly|daily|weekly|monthly)-([0-9]{8}T[0-9]{4}Z)$`)
|
|
|
|
// SnapshotSetArgv sets a user property on a snapshot; only mesh:schedule is ever set by the module.
|
|
func SnapshotSetArgv(snapshot, kind string) []string {
|
|
return []string{"set", "mesh:schedule=" + kind, "--", snapshot}
|
|
}
|
|
|
|
// LedgerEntry is one instance of the ledger `snapshots` (A-30): the ZFS snapshot the run took.
|
|
type LedgerEntry struct {
|
|
Dataset string `json:"dataset"`
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
At string `json:"at"`
|
|
}
|
|
|
|
// LedgerInstance is the instance path of a scheduled snapshot: `snapshot/<kind>-<UTC stamp>-<dataset>`, the
|
|
// dataset's `/` written `~`.
|
|
func LedgerInstance(kind, stamp, dataset string) string {
|
|
return "snapshot/" + kind + "-" + stamp + "-" + strings.ReplaceAll(dataset, "/", "~")
|
|
}
|
|
|
|
// InstanceOf is the instance of a record's snapshot, refusing a name that is not a scheduled snapshot's.
|
|
func InstanceOf(dataset, name string) (string, error) {
|
|
m := meshSnapshot.FindStringSubmatch(name)
|
|
if m == nil {
|
|
return "", fmt.Errorf("%q is not a scheduled snapshot's name (mesh-<kind>-<UTC stamp>): the ledger holds only the run's", name)
|
|
}
|
|
if dataset == "" || strings.ContainsAny(dataset, "@~ ") {
|
|
return "", fmt.Errorf("%q is not a dataset's name", dataset)
|
|
}
|
|
return LedgerInstance(m[1], m[2], dataset), nil
|
|
}
|
|
|
|
// ledgerEntries reads the ledger through the module's tool zfs_snapshots_schedule.
|
|
func (m *Machine) ledgerEntries() ([]LedgerEntry, error) {
|
|
if m.Call == nil {
|
|
return nil, fmt.Errorf("this process reaches no tool, so the ledger cannot be read")
|
|
}
|
|
raw, err := m.Call(ToolSchedule, map[string]any{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var ans struct {
|
|
Ledger []LedgerEntry `json:"ledger"`
|
|
}
|
|
if err := json.Unmarshal(raw, &ans); err != nil {
|
|
return nil, fmt.Errorf("zfs_snapshots_schedule answered what is no ledger: %v", err)
|
|
}
|
|
return ans.Ledger, nil
|
|
}
|
|
|
|
// record tells the ledger of a taken or removed snapshot through zfs_snapshots_record.
|
|
func (m *Machine) recordLedger(op string, e LedgerEntry) error {
|
|
if m.Call == nil {
|
|
return fmt.Errorf("this process reaches no tool, so %s was not recorded", e.Name)
|
|
}
|
|
_, err := m.Call(ToolRecord, map[string]any{"operation": op, "dataset": e.Dataset, "name": e.Name, "kind": e.Kind, "at": e.At})
|
|
return err
|
|
}
|
|
|
|
// keepsOf are the kinds with a keep count above 0, in the order they are said.
|
|
func keepsOf(k *Keeps) []string {
|
|
var out []string
|
|
for _, kind := range keepKinds {
|
|
if k.Of(kind) > 0 {
|
|
out = append(out, kind)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// dueKinds are the kinds due now: the calendar's moment of the kind, and no entry of the kind since its period began.
|
|
func dueKinds(k *Keeps, now time.Time, entries []LedgerEntry) []string {
|
|
day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
|
var due []string
|
|
for _, kind := range keepsOf(k) {
|
|
start := day
|
|
switch kind {
|
|
case "hourly":
|
|
start = now.Truncate(time.Hour)
|
|
case "weekly":
|
|
if now.Weekday() != time.Monday {
|
|
continue
|
|
}
|
|
case "monthly":
|
|
if now.Day() != 1 {
|
|
continue
|
|
}
|
|
}
|
|
var last time.Time
|
|
for _, e := range entries {
|
|
if at, err := time.Parse(time.RFC3339, e.At); err == nil && e.Kind == kind && at.After(last) {
|
|
last = at
|
|
}
|
|
}
|
|
if last.Before(start) {
|
|
due = append(due, kind)
|
|
}
|
|
}
|
|
return due
|
|
}
|
|
|
|
// ledgerCap is the ledger's bound: the keep counts' sum times two instances per dataset (§26).
|
|
func ledgerCap(k *Keeps, datasets int) int {
|
|
return (k.Hourly + k.Daily + k.Weekly + k.Monthly) * 2 * datasets
|
|
}
|
|
|
|
func (m *Machine) skippedSnapshots(person bool, pool string, capacity int, reason string) {
|
|
kind := EventSkippedSched
|
|
if person {
|
|
kind = EventSkippedPerson
|
|
}
|
|
var failed []error
|
|
if err := m.say(kind, map[string]any{"pool": pool, "capacity": capacity, "reason": reason}); err != nil {
|
|
m.failure(&failed, kind, err)
|
|
}
|
|
}
|
|
|
|
// filesystems are the pool's filesystems, never a volume.
|
|
func (m *Machine) filesystems(ctx context.Context, pool string) ([]string, int, error) {
|
|
ds, err := m.Datasets(ctx)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
var out []string
|
|
for _, d := range ds {
|
|
if d.Type == "filesystem" && (d.Name == pool || strings.HasPrefix(d.Name, pool+"/")) {
|
|
out = append(out, d.Name)
|
|
}
|
|
}
|
|
capacity := 0
|
|
ps, err := m.Pools(ctx)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
for _, p := range ps {
|
|
if p.Name == pool {
|
|
capacity = p.Capacity
|
|
}
|
|
}
|
|
return out, capacity, nil
|
|
}
|
|
|
|
// TakeScheduled takes the due ZFS snapshots (every kind with a keep count when force: a person's act, by
|
|
// them). It answers what it did, and a failed command as the error.
|
|
func (m *Machine) TakeScheduled(ctx context.Context, c *Config, force bool, by string) ([]string, error) {
|
|
if c.Snapshots == nil || c.Snapshots.None || c.Pool == "" {
|
|
return nil, nil
|
|
}
|
|
person := force
|
|
taken := EventTakenSchedule
|
|
if person {
|
|
taken = EventTakenPerson
|
|
} else {
|
|
by = "schedule"
|
|
}
|
|
fs, capacity, err := m.filesystems(ctx, c.Pool)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if m.Paused() {
|
|
m.skippedSnapshots(person, c.Pool, capacity, "paused")
|
|
return []string{"snapshots paused"}, nil
|
|
}
|
|
entries, err := m.ledgerEntries()
|
|
if err != nil {
|
|
m.skippedSnapshots(person, c.Pool, capacity, "ledger-unavailable: "+err.Error())
|
|
return []string{"no snapshot taken: the ledger cannot be read"}, nil
|
|
}
|
|
due := keepsOf(c.Snapshots)
|
|
if !force {
|
|
due = dueKinds(c.Snapshots, m.Now(), entries)
|
|
}
|
|
if len(due) == 0 || len(fs) == 0 {
|
|
return nil, nil
|
|
}
|
|
if capacity > FullPercent {
|
|
m.skippedSnapshots(person, c.Pool, capacity, "full")
|
|
return []string{fmt.Sprintf("no snapshot taken: the pool is %d %% full", capacity)}, nil
|
|
}
|
|
if len(entries)+len(due)*len(fs) > ledgerCap(c.Snapshots, len(fs)) {
|
|
m.skippedSnapshots(person, c.Pool, capacity, "ledger-full")
|
|
return []string{"no snapshot taken: the ledger is full"}, nil
|
|
}
|
|
now := m.Now().UTC()
|
|
var said []string
|
|
var failed []error
|
|
for _, kind := range due {
|
|
for _, ds := range fs {
|
|
name := fmt.Sprintf("%s%s-%s", MeshPrefix, kind, now.Format(StampForm))
|
|
full := ds + "@" + name
|
|
if _, err := m.AsRoot(ctx, "zfs", SnapshotArgv(ds, name)...); err != nil {
|
|
failed = append(failed, fmt.Errorf("%s: %v", full, err))
|
|
continue
|
|
}
|
|
if _, err := m.AsRoot(ctx, "zfs", SnapshotSetArgv(full, kind)...); err != nil {
|
|
failed = append(failed, fmt.Errorf("%s was taken, and mesh:schedule could not be set: %v", full, err))
|
|
continue
|
|
}
|
|
e := LedgerEntry{Dataset: ds, Name: name, Kind: kind, At: now.Format(time.RFC3339)}
|
|
if err := m.recordLedger("taken", e); err != nil {
|
|
failed = append(failed, fmt.Errorf("%s was taken, and the ledger did not take it: %v", full, err))
|
|
continue
|
|
}
|
|
if err := m.say(taken, map[string]any{"snapshot": full, "by": by}); err != nil {
|
|
m.failure(&failed, taken, err)
|
|
}
|
|
said = append(said, "took "+full)
|
|
}
|
|
}
|
|
return said, joinErrors(failed)
|
|
}
|
|
|
|
// LedgerName is the ledger's local name in the module's `requires`.
|
|
const LedgerName = "snapshots"
|
|
|
|
// LedgerStore is the ledger `snapshots` as the tools bundle reaches it (A-20, A-30).
|
|
type LedgerStore interface {
|
|
// Apply applies an operation to an instance; call is the id of the tool call it serves.
|
|
Apply(instance, operation string, body any, call string) error
|
|
// Entries are every instance's state.
|
|
Entries() ([]LedgerEntry, error)
|
|
}
|
|
|
|
// Record applies the run's `taken` or `removed` to the ledger, refusing a snapshot not named mesh- (A-20).
|
|
func (m *Machine) Record(op string, e LedgerEntry, call string) error {
|
|
if op != "taken" && op != "removed" {
|
|
return fmt.Errorf("%q is not an operation of the ledger: taken or removed", op)
|
|
}
|
|
instance, err := InstanceOf(e.Dataset, e.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if m.Ledger == nil {
|
|
return fmt.Errorf("this process reaches no ledger")
|
|
}
|
|
if op == "removed" {
|
|
return m.Ledger.Apply(instance, op, struct{}{}, call)
|
|
}
|
|
return m.Ledger.Apply(instance, op, e, call)
|
|
}
|