package main import ( "context" "errors" "fmt" "regexp" "strings" ) // The apply's cache and log devices (design 42 zfs-owns-zfs §17). // // From `zpool status -P -L ` the apply knows each entry's kernel path, from `zpool status -P ` the // name the pool holds it by, and from the link under /dev/disk/by-id each declared name's kernel path. Then, in // this order: an entry named by its kernel name that is a declared device is removed and added again by its // stable name; a declared device not in the pool is added after its checks, never with -f; only while the list // is set and not empty is a cache entry it does not name removed; and a log vdev it does not name, one to // rename, or the log when `log-mirror` changes is removed only while the pool is healthy and idle, else it is // a wait. A declared device that is not present is a wait. // ByIDDir is where a declared device's stable name is a link. const ByIDDir = "/dev/disk/by-id/" // DeviceStatusArgv reads one pool's status with the paths its vdevs are held by (real false) or the kernel // paths they resolve to (real true). func DeviceStatusArgv(pool string, real bool) []string { a := []string{"status", "-P"} if real { a = append(a, "-L") } return append(a, "--", pool) } // AllRealStatusArgv reads every imported pool's vdevs by kernel path. func AllRealStatusArgv() []string { return []string{"status", "-P", "-L"} } // RemoveArgv removes one cache or log entry of the pool. func RemoveArgv(pool, entry string) []string { return []string{"remove", "--", pool, entry} } // AddArgv adds cache or log devices by their paths; the log as one mirror when mirror is true. Never -f. func AddArgv(pool, role string, mirror bool, paths ...string) []string { a := []string{"add", "--", pool, role} if mirror { a = append(a, "mirror") } return append(a, paths...) } // WipefsArgv lists a device's signatures and writes nothing. func WipefsArgv(device string) []string { return []string{"--no-act", device} } // ZdbLabelArgv reads a device's ZFS label. func ZdbLabelArgv(device string) []string { return []string{"-l", device} } // devEntry is one cache or log device of the pool. type devEntry struct { Held string // the path the pool holds it by Real string // its kernel path Role string // cache or log Vdev string // the top-level vdev it is in ("" for a device of its own) } // logVdev is one top-level vdev of the log: a device of its own, or a mirror. type logVdev struct { Name string Mirror bool Members []devEntry } // poolDevices is what the apply knows of the pool's cache and log. type poolDevices struct { Cache []devEntry Log []logVdev Healthy bool // ONLINE, no error on any device, no scrub or resilver running Why string // what holds it from being healthy } func (p *poolDevices) members() []devEntry { out := append([]devEntry{}, p.Cache...) for _, v := range p.Log { out = append(out, v.Members...) } return out } func (p *poolDevices) inPool(real string) *devEntry { for _, e := range p.members() { if e.Real == real { return &e } } return nil } func findPool(out, pool string) (*PoolStatus, error) { ps, err := ParseStatus(out) if err != nil { return nil, err } for i := range ps { if ps[i].Name == pool { return &ps[i], nil } } return nil, fmt.Errorf("zpool status does not show the pool %s", pool) } func sectionOf(p *PoolStatus, role string) *Device { if p.Root == nil { return nil } for _, c := range p.Root.Children { if c.Kind == "section" && c.Role == role { return c } } return nil } func deviceErrors(d *Device) uint64 { n := d.Errors() for _, c := range d.Children { n += deviceErrors(c) } return n } // devicesOf reads the pool's cache and log, held-by and real read side by side. func (m *Machine) devicesOf(ctx context.Context, pool string) (*poolDevices, error) { var st [2]*PoolStatus for i, real := range []bool{false, true} { out, err := m.ask(ctx, "zpool", DeviceStatusArgv(pool, real)...) if err != nil { return nil, err } if st[i], err = findPool(out, pool); err != nil { return nil, err } } held, real := st[0], st[1] pd := &poolDevices{Healthy: true} switch { case real.State != "ONLINE": pd.Healthy, pd.Why = false, "the pool is "+real.State case real.Root != nil && deviceErrors(real.Root) > 0: pd.Healthy, pd.Why = false, "a device of the pool has read, write or checksum errors" } if busy := scanBusy(real.Scans); busy != "" { pd.Healthy, pd.Why = false, busy } for _, role := range []string{"cache", "log"} { hs, rs := sectionOf(held, role), sectionOf(real, role) if hs == nil || rs == nil { continue } if len(hs.Children) != len(rs.Children) { return nil, fmt.Errorf("zpool status -P and -P -L list a different %s", role) } for i, hc := range hs.Children { rc := rs.Children[i] if len(hc.Children) != len(rc.Children) { return nil, fmt.Errorf("zpool status -P and -P -L list a different %s", role) } if len(hc.Children) == 0 { e := devEntry{Held: hc.Name, Real: rc.Name, Role: role} if role == "cache" { pd.Cache = append(pd.Cache, e) } else { pd.Log = append(pd.Log, logVdev{Name: hc.Name, Members: []devEntry{e}}) } continue } v := logVdev{Name: hc.Name, Mirror: true} for j, hm := range hc.Children { v.Members = append(v.Members, devEntry{Held: hm.Name, Real: rc.Children[j].Name, Role: role, Vdev: hc.Name}) } if role == "log" { pd.Log = append(pd.Log, v) } else { return nil, fmt.Errorf("the cache holds a vdev %s, which a cache never is", hc.Name) } } } return pd, nil } // declared is one device the config names. type declared struct { Name, Path, Real, Role string Present bool } func (m *Machine) resolveDeclared(names []string, role string) []declared { var out []declared for _, n := range names { d := declared{Name: n, Path: ByIDDir + n, Role: role} if m.Resolve != nil { if r, err := m.Resolve(d.Path); err == nil { d.Real, d.Present = r, true } } out = append(out, d) } return out } func byID(held string) bool { return strings.HasPrefix(held, ByIDDir) } var labelName = regexp.MustCompile(`(?m)^\s*name:\s*'([^']*)'`) // partOf says whether entry is device itself or one of its partitions. func partOf(entry, device string) bool { rest, ok := strings.CutPrefix(entry, device) if !ok { return false } if rest == "" { return true } rest = strings.TrimPrefix(rest, "p") return rest != "" && strings.Trim(rest, "0123456789") == "" } // checkDevice is the refusal's reason for a device to add, "" when it may be added (§17): a partition table or // a file-system signature refuses it, but for a zfs_member signature judged by its label (this pool's is // allowed, another's refused), and a member of another imported pool is refused. func (m *Machine) checkDevice(ctx context.Context, pool string, d declared, others []string) (string, error) { for _, o := range others { if partOf(o, d.Real) || partOf(d.Real, o) { return d.Name + " is a member of another imported pool", nil } } r, err := m.AsRoot(ctx, "wipefs", WipefsArgv(d.Path)...) if err != nil { return "", err } zfsMember := false for _, l := range lines(r.Out) { f := strings.Fields(l) if len(f) == 0 || f[0] == "DEVICE" { continue } if len(f) < 3 { return d.Name + " shows a signature wipefs names no type for: " + l, nil } if f[2] != "zfs_member" { return fmt.Sprintf("%s holds a %s signature or partition table", d.Name, f[2]), nil } zfsMember = true } if !zfsMember { return "", nil } z, err := m.AsRoot(ctx, "zdb", ZdbLabelArgv(d.Path)...) if err != nil { return d.Name + " holds a ZFS label that cannot be read: " + err.Error(), nil } lm := labelName.FindStringSubmatch(z.Out) switch { case lm == nil: return d.Name + " holds a ZFS label that names no pool", nil case lm[1] != pool: return fmt.Sprintf("%s holds the label of the pool %s, not %s", d.Name, lm[1], pool), nil } return "", nil } // otherPoolDevices are the kernel paths of every imported pool's vdevs but the pool's own. func (m *Machine) otherPoolDevices(ctx context.Context, pool string) ([]string, error) { out, err := m.ask(ctx, "zpool", AllRealStatusArgv()...) if err != nil { return nil, err } ps, err := ParseStatus(out) if err != nil { return nil, err } var paths []string var walk func(d *Device) walk = func(d *Device) { for _, c := range d.Children { if strings.HasPrefix(c.Name, "/dev/") { paths = append(paths, c.Name) } walk(c) } } for _, p := range ps { if p.Name != pool && p.Root != nil { walk(p.Root) } } return paths, nil } // ApplyDevices converges the cache and log devices the config names; it answers every failed command as one // error. An empty list is not managed, so it removes nothing. func (m *Machine) ApplyDevices(ctx context.Context, c *Config) error { if c.Pool == "" || (len(c.CacheDevices) == 0 && len(c.LogDevices) == 0) { return nil } var failed []error fail := func(key string, err error) { m.failure(&failed, key, err) } others, err := m.otherPoolDevices(ctx, c.Pool) if err != nil { fail("cache-devices", err) return errors.Join(failed...) } for _, role := range []string{"cache", "log"} { names := c.CacheDevices if role == "log" { names = c.LogDevices } if len(names) == 0 { continue } key := role + "-devices" if err := m.applyRole(ctx, c, role, key, m.resolveDeclared(names, role), others); err != nil { fail(key, err) } } return errors.Join(failed...) } func (m *Machine) applyRole(ctx context.Context, c *Config, role, key string, want []declared, others []string) error { pd, err := m.devicesOf(ctx, c.Pool) if err != nil { return err } before := m.rolesSaid(pd, role) for _, d := range want { if !d.Present { return m.Waiting(key, "the device "+d.Name+" to appear under "+ByIDDir) } } if role == "log" { if bad, _ := badLog(c, want, pd); len(bad) > 0 && !pd.Healthy { return m.Waiting(key, "the pool to be ONLINE without errors and with no scrub or resilver ("+pd.Why+")") } } // Checks first, so nothing is removed that cannot be put back. for _, d := range want { if e := pd.inPool(d.Real); e != nil && byID(e.Held) { continue } var out []string if pd.inPool(d.Real) == nil { out = others } why, err := m.checkDevice(ctx, c.Pool, d, out) if err != nil { return err } if why != "" { return m.Refused(Refusal{Key: key, Value: d.Name, Reason: why}) } } var changes []string if role == "cache" { changes, err = m.applyCache(ctx, c, want, pd) } else { changes, err = m.applyLog(ctx, c, want, pd) } if err != nil { return err } after, err := m.devicesOf(ctx, c.Pool) if err != nil { return err } for _, d := range want { if e := after.inPool(d.Real); e == nil || !byID(e.Held) { if len(changes) == 0 { continue } return fmt.Errorf("read back: %s is not in the pool by its stable name after the change", d.Name) } } if len(changes) == 0 { return m.keepAsSet(key, before) } return m.Changed("pool", key, before, strings.Join(changes, "; "), m.rolesSaid(after, role)) } func (m *Machine) rolesSaid(pd *poolDevices, role string) string { var held []string for _, e := range pd.members() { if e.Role == role { held = append(held, e.Held) } } return strings.Join(held, " ") } func (m *Machine) applyCache(ctx context.Context, c *Config, want []declared, pd *poolDevices) ([]string, error) { var changes []string for _, d := range want { if e := pd.inPool(d.Real); e != nil && e.Role == "cache" && !byID(e.Held) { if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, e.Held)...); err != nil { return changes, err } if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "cache", false, d.Path)...); err != nil { return changes, err } changes = append(changes, fmt.Sprintf("%s re-added by its stable name %s", e.Held, d.Name)) } } var adds []string var names []string for _, d := range want { if pd.inPool(d.Real) == nil { adds, names = append(adds, d.Path), append(names, d.Name) } } if len(adds) > 0 { if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "cache", false, adds...)...); err != nil { return changes, err } changes = append(changes, "added "+strings.Join(names, " ")) } for _, e := range pd.Cache { if !declaredReal(want, e.Real) { if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, e.Held)...); err != nil { return changes, err } changes = append(changes, "removed "+e.Held) } } return changes, nil } func declaredReal(want []declared, real string) bool { for _, d := range want { if d.Real == real { return true } } return false } // badLog is the log vdevs that must go: one holding a device the config does not name or one held by its // kernel name, any when log-mirror asks the other form, and a mirror whose members are not the declared pair. func badLog(c *Config, want []declared, pd *poolDevices) (bad []logVdev, mirrorChange bool) { for _, v := range pd.Log { b := false for _, e := range v.Members { if !declaredReal(want, e.Real) || !byID(e.Held) { b = true } } if c.LogMirror != nil { if *c.LogMirror && (!v.Mirror || len(v.Members) != len(want)) { b, mirrorChange = true, true } if !*c.LogMirror && v.Mirror { b, mirrorChange = true, true } } if b { bad = append(bad, v) } } return bad, mirrorChange } func (m *Machine) applyLog(ctx context.Context, c *Config, want []declared, pd *poolDevices) ([]string, error) { var changes []string bad, mirrorChange := badLog(c, want, pd) if len(bad) > 0 { for _, v := range bad { if _, err := m.AsRoot(ctx, "zpool", RemoveArgv(c.Pool, v.Name)...); err != nil { return changes, err } note := "removed " + v.Name if mirrorChange { note += " (a log-mirror change: a removal and a re-add)" } changes = append(changes, note) } var err error if pd, err = m.devicesOf(ctx, c.Pool); err != nil { return changes, err } } var missing []declared for _, d := range want { if pd.inPool(d.Real) == nil { missing = append(missing, d) } } if len(missing) == 0 { return changes, nil } var paths, names []string for _, d := range missing { paths, names = append(paths, d.Path), append(names, d.Name) } mirror := c.LogMirror != nil && *c.LogMirror if mirror && len(missing) == len(want) { if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "log", true, paths...)...); err != nil { return changes, err } } else { for _, p := range paths { if _, err := m.AsRoot(ctx, "zpool", AddArgv(c.Pool, "log", false, p)...); err != nil { return changes, err } } } return append(changes, "added "+strings.Join(names, " ")), nil }