package main import ( "bufio" "context" "fmt" "io" "os/exec" "path/filepath" "sort" "strings" "syscall" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // What the event follower does besides the journal (design 42 zfs-owns-zfs §18, §27; A-10, A-16). // // **A wait is ended by an event.** The follower, which runs from boot, starts the apply unit once at its own // start and when it hears a scrub or resilver finish (`scrub_finish`, `resilver_finish`) or a declared device // appear (a udev `add` of a block device whose link under /dev/disk/by-id is declared), through // `/node-service-manager.start`. It also keeps the findings: one key `pool..` per device of // the pool, `pool..pool` for the pool, not ONLINE, its value the state, deleted when the device is ONLINE // again or leaves the pool. // ApplyUnit is the unit of the apply process (`zfs-config-apply`). const ApplyUnit = "zfs-config-apply.service" // UdevArgv follows the kernel's block device events. func UdevArgv() []string { return []string{"monitor", "--udev", "--subsystem-match=block"} } // StartApply starts the apply unit through the service manager. func (m *Machine) StartApply() error { if m.Call == nil { return fmt.Errorf("this process reaches no seat, so the apply was not started") } _, err := m.Call("seat:node-service-manager.start@"+m.Node, map[string]any{"scope": "system", "unit": ApplyUnit}) return err } // EndsAWait says whether an event of ZFS's ends a wait of the apply. func EndsAWait(class string) bool { return strings.HasSuffix(class, ".scrub_finish") || strings.HasSuffix(class, ".resilver_finish") } // ChangesAFinding says whether an event of ZFS's may change a device's state. func ChangesAFinding(class string) bool { return strings.HasSuffix(class, ".statechange") || strings.HasPrefix(class, "resource.fs.zfs.") || strings.HasSuffix(class, ".config_sync") || EndsAWait(class) } // DeclaredAdded says whether a line of `udevadm monitor` is the add of a block device a declared name links to. // The declared names are read from the config file at each add, the links resolved now. func (m *Machine) DeclaredAdded(line string) bool { f := strings.Fields(line) if len(f) < 4 || f[0] != "UDEV" || f[2] != "add" { return false } added := filepath.Base(f[3]) text, err := m.ReadFile(ConfigFile) if err != nil { return false } c, refused := ParseConfig(text, m.memMiB()) if len(refused) > 0 { return false } for _, d := range append(m.resolveDeclared(c.CacheDevices, "cache"), m.resolveDeclared(c.LogDevices, "log")...) { if d.Present && partOf(added, filepath.Base(d.Real)) { return true } } return false } // FollowUdev reads `udevadm monitor` and starts the apply on the add of a declared device. func (m *Machine) FollowUdev(r io.Reader) error { sc := bufio.NewScanner(r) for sc.Scan() { if m.DeclaredAdded(sc.Text()) { if err := m.StartApply(); err != nil { return err } } } return sc.Err() } // StartUdev starts `udevadm monitor` and follows it until the context ends. func (m *Machine) StartUdev(ctx context.Context) error { cmd := exec.Command("udevadm", UdevArgv()...) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} out, err := cmd.StdoutPipe() if err != nil { return err } if err := cmd.Start(); err != nil { return err } go func() { <-ctx.Done() _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }() ferr := m.FollowUdev(out) _ = cmd.Wait() return ferr } // FindingsOf are the findings `zpool status -P` shows now: `pool..` to state, for a pool or a device not // ONLINE, the device by its own name, the last element of its path (A-341). func FindingsOf(out string) (map[string]string, error) { ps, err := ParseStatus(out) if err != nil { return nil, err } found := map[string]string{} var walk func(pool string, d *Device) walk = func(pool string, d *Device) { for _, c := range d.Children { if c.Kind != "section" && c.State != "" && c.State != "ONLINE" && c.State != "AVAIL" && c.State != "INUSE" { found["pool."+pool+"."+filepath.Base(c.Name)] = c.State } walk(pool, c) } } for _, p := range ps { if p.State != "ONLINE" { found["pool."+p.Name+".pool"] = p.State } if p.Root != nil { walk(p.Name, p.Root) } } return found, nil } // FindingsObserver is what the findings are written as: the module is their observer (A-32). const FindingsObserver = "zfs" // FindingKind is the one kind of the follower's findings, a pool or a device not ONLINE (A-341): each is its own key // by its id, `zfs..` (`zfs..pool` for the pool's own state), under the module's scope. const FindingKind = "not-online" // FindingsWriter is what the follower says its findings through (A-339): G's writer of the seat mesh-conditions, which // keeps each in the state `findings` and says it on the module's finding stream. type FindingsWriter interface { Write(ctx context.Context, f seat.Finding) error Clear(ctx context.Context, scope, id, kind, why string) error } // findingOf is the finding of one device or pool not ONLINE, by the name FindingsOf gives it, `pool..` // (A-341): its id `zfs..`, its subject the module on the node. func findingOf(name, state, node string, at time.Time) seat.Finding { short := strings.TrimPrefix(name, "pool.") return seat.Finding{Scope: meshconditionsclient.ScopeModule, ID: FindingsObserver + "." + short, Kind: FindingKind, Observer: FindingsObserver, Subject: seat.Subject{Scope: meshconditionsclient.ScopeModule, ID: FindingsObserver, Machine: node}, Severity: meshconditionsclient.Warning, Source: FindingsObserver + "." + FindingKind, Summary: fmt.Sprintf("%s is %s", short, state), Headline: fmt.Sprintf("ZFS %s is %s", short, state), Explanation: "ZFS reports this pool or device not ONLINE; zfs_pool_layout and zfs_pools say more. The finding clears when it is ONLINE again or leaves the pool.", At: at.UTC()} } // ObserveFindings are the findings `zpool status -P` shows now, each as the writer says it (§27, G §384: judged anew // from the module's own truth at its start and on each ask). func (m *Machine) ObserveFindings(ctx context.Context) ([]seat.Finding, error) { out, err := m.ask(ctx, "zpool", StatusPathsArgv()...) if err != nil { return nil, err } now, err := FindingsOf(out) if err != nil { return nil, err } var found []seat.Finding for _, name := range sortedKeys(now) { found = append(found, findingOf(name, now[name], m.Node, m.Now())) } return found, nil } // SyncFindings reads `zpool status -P` against the keys of the state `findings`, writes what is new or changed (the // writer says only a change), and clears what is gone, each through the writer. func (m *Machine) SyncFindings(ctx context.Context) error { if m.States == nil || m.Findings == nil { return fmt.Errorf("this process keeps no state, so the findings were not read") } found, err := m.ObserveFindings(ctx) if err != nil { return err } keys, err := m.States.Keys(StateFindings) if err != nil { return err } wanted := map[string]bool{} for _, f := range found { wanted[f.ID] = true if err := m.Findings.Write(ctx, f); err != nil { return err } } prefix := meshconditionsclient.ScopeModule + "." + FindingsObserver + "." sort.Strings(keys) for _, k := range keys { rest, mine := strings.CutPrefix(k, meshconditionsclient.ScopeModule+".") id, kind := rest, "" if i := strings.LastIndex(rest, "."); i > 0 { id, kind = rest[:i], rest[i+1:] } if !mine || !strings.HasPrefix(k, prefix) || kind != FindingKind || wanted[id] { continue } if err := m.Findings.Clear(ctx, meshconditionsclient.ScopeModule, id, FindingKind, "ONLINE again or left the pool"); err != nil { return err } } return nil } // StatusPathsArgv reads every pool's status with the paths its devices are held by. func StatusPathsArgv() []string { return []string{"status", "-P"} }