package main import ( "bufio" "context" "fmt" "io" "os/exec" "path/filepath" "sort" "strings" "syscall" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/views" ) // What the event follower does besides the journal (design 42 zfs-owns-zfs §18, §27; A-10, A-16). // // **A wait is ended by an event.** The follower, which runs from boot, starts the apply unit once at its own // start and when it hears a scrub or resilver finish (`scrub_finish`, `resilver_finish`) or a declared device // appear (a udev `add` of a block device whose link under /dev/disk/by-id is declared), through // `/node-service-manager.start`. It also keeps the findings: one key `pool..` per device of // the pool, `pool..pool` for the pool, not ONLINE, its value the state, deleted when the device is ONLINE // again or leaves the pool. // ApplyUnit is the unit of the apply process (`zfs-config-apply`). const ApplyUnit = "zfs-config-apply.service" // UdevArgv follows the kernel's block device events. func UdevArgv() []string { return []string{"monitor", "--udev", "--subsystem-match=block"} } // StartApply starts the apply unit through the service manager. func (m *Machine) StartApply() error { if m.Call == nil { return fmt.Errorf("this process reaches no seat, so the apply was not started") } _, err := m.Call("seat:node-service-manager.start@"+m.Node, map[string]any{"scope": "system", "unit": ApplyUnit}) return err } // EndsAWait says whether an event of ZFS's ends a wait of the apply. func EndsAWait(class string) bool { return strings.HasSuffix(class, ".scrub_finish") || strings.HasSuffix(class, ".resilver_finish") } // ChangesAFinding says whether an event of ZFS's may change a device's state. func ChangesAFinding(class string) bool { return strings.HasSuffix(class, ".statechange") || strings.HasPrefix(class, "resource.fs.zfs.") || strings.HasSuffix(class, ".config_sync") || EndsAWait(class) } // DeclaredAdded says whether a line of `udevadm monitor` is the add of a block device a declared name links to. // The declared names are read from the config file at each add, the links resolved now. func (m *Machine) DeclaredAdded(line string) bool { f := strings.Fields(line) if len(f) < 4 || f[0] != "UDEV" || f[2] != "add" { return false } added := filepath.Base(f[3]) text, err := m.ReadFile(ConfigFile) if err != nil { return false } c, refused := ParseConfig(text, m.memMiB()) if len(refused) > 0 { return false } for _, d := range append(m.resolveDeclared(c.CacheDevices, "cache"), m.resolveDeclared(c.LogDevices, "log")...) { if d.Present && partOf(added, filepath.Base(d.Real)) { return true } } return false } // FollowUdev reads `udevadm monitor` and starts the apply on the add of a declared device. func (m *Machine) FollowUdev(r io.Reader) error { sc := bufio.NewScanner(r) for sc.Scan() { if m.DeclaredAdded(sc.Text()) { if err := m.StartApply(); err != nil { return err } } } return sc.Err() } // StartUdev starts `udevadm monitor` and follows it until the context ends. func (m *Machine) StartUdev(ctx context.Context) error { cmd := exec.Command("udevadm", UdevArgv()...) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} out, err := cmd.StdoutPipe() if err != nil { return err } if err := cmd.Start(); err != nil { return err } go func() { <-ctx.Done() _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }() ferr := m.FollowUdev(out) _ = cmd.Wait() return ferr } // FindingsOf are the findings `zpool status -P` shows now: key to state, for a pool or a device not ONLINE. func FindingsOf(out string) (map[string]string, error) { ps, err := ParseStatus(out) if err != nil { return nil, err } found := map[string]string{} var walk func(pool string, d *Device) walk = func(pool string, d *Device) { for _, c := range d.Children { if c.Kind != "section" && c.State != "" && c.State != "ONLINE" && c.State != "AVAIL" && c.State != "INUSE" { found["pool."+pool+"."+c.Name] = c.State } walk(pool, c) } } for _, p := range ps { if p.State != "ONLINE" { found["pool."+p.Name+".pool"] = p.State } if p.Root != nil { walk(p.Name, p.Root) } } return found, nil } // FindingsObserver is what the findings are written as: the module is their observer, as every observer on the mesh // writes its own state `findings` with mesh-sdk's views.Finding and the seat mesh-conditions' writer (A-32). const FindingsObserver = "zfs" // findingOf is the finding of one device or pool not ONLINE: the module is its subject, the pool and device its // kind, `pool..`, so each is a key of its own and goes when the device is ONLINE again or gone. func findingOf(kind, state string, at time.Time) views.Finding { name := strings.TrimPrefix(kind, "pool.") return views.Finding{Observer: FindingsObserver, Kind: kind, ID: FindingsObserver, Summary: fmt.Sprintf("%s is %s", name, state), Headline: fmt.Sprintf("ZFS %s is %s", name, state), Explanation: "ZFS reports this pool or device not ONLINE; zfs_pool_layout and zfs_pools say more. The finding clears when it is ONLINE again or leaves the pool.", At: at} } // SyncFindings reads `zpool status -P` against the keys of the state `findings`, writes what is new or changed, // and clears what is gone, each through the writer. func (m *Machine) SyncFindings(ctx context.Context) error { if m.States == nil || m.Findings == nil { return fmt.Errorf("this process keeps no state, so the findings were not read") } out, err := m.ask(ctx, "zpool", StatusPathsArgv()...) if err != nil { return err } now, err := FindingsOf(out) if err != nil { return err } keys, err := m.States.Keys(StateFindings) if err != nil { return err } prefix := strings.TrimSuffix(meshconditionsclient.Key(meshconditionsclient.ScopeModule, FindingsObserver, "x"), "x") wanted := map[string]bool{} for _, kind := range sortedKeys(now) { f := findingOf(kind, now[kind], m.Now()) key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, FindingsObserver, kind) wanted[key] = true if raw, ok, _ := m.States.Get(StateFindings, key); ok && strings.Contains(string(raw), `"summary":"`+f.Summary+`"`) { continue } if err := m.Findings.Put(f); err != nil { return err } } sort.Strings(keys) for _, k := range keys { kind, mine := strings.CutPrefix(k, prefix) if !mine || wanted[k] { continue } if err := m.Findings.Delete(views.Finding{ID: FindingsObserver, Kind: kind}); err != nil { return err } } return nil } // StatusPathsArgv reads every pool's status with the paths its devices are held by. func StatusPathsArgv() []string { return []string{"status", "-P"} }