mesh/delivery merged: source.main moved to novox/mesh-catalog@6ba73d4e2a4f
mesh/delivery-group group feat/47-G-join12 merged: every member is merged
mesh/check failed (2): mesh-delivery-checker: no published of the tagged version of go-meshconditions of me…; and 1 more | 53351dd30df2 on b1125c02cfe6
mesh-conditions keeps G's form: the keeper reads no observer's state (no reads, no Observers list), so main's zfs entry there goes. The zfs follower says its findings through G's findings writer over its keeper: the keeper gains the publish of the writer's three events on the finding stream finding-events and the hear of mesh-conditions.findings-asked on its own durable, nothing else; each pool or device not ONLINE is scope module, id zfs.<pool>.<device> (the device's own name), kind not-online, on the node. The manifest asks its state findings by template in requires (G §529, §577), and gains the finding-events stream and the consumes, and the keeper's bus the three events and the consumes. Builds: design 47 G §383, §384, §387, §529, §577, A-339, A-340, A-341, A-342; design 42 zfs-owns-zfs §27, A-32, A-37, A-39, A-41 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
225 lines
8.0 KiB
Go
225 lines
8.0 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
)
|
|
|
|
// What the event follower does besides the journal (design 42 zfs-owns-zfs §18, §27; A-10, A-16).
|
|
//
|
|
// **A wait is ended by an event.** The follower, which runs from boot, starts the apply unit once at its own
|
|
// start and when it hears a scrub or resilver finish (`scrub_finish`, `resilver_finish`) or a declared device
|
|
// appear (a udev `add` of a block device whose link under /dev/disk/by-id is declared), through
|
|
// `<node>/node-service-manager.start`. It also keeps the findings: one key `pool.<pool>.<device>` per device of
|
|
// the pool, `pool.<pool>.pool` for the pool, not ONLINE, its value the state, deleted when the device is ONLINE
|
|
// again or leaves the pool.
|
|
|
|
// ApplyUnit is the unit of the apply process (`zfs-config-apply`).
|
|
const ApplyUnit = "zfs-config-apply.service"
|
|
|
|
// UdevArgv follows the kernel's block device events.
|
|
func UdevArgv() []string { return []string{"monitor", "--udev", "--subsystem-match=block"} }
|
|
|
|
// StartApply starts the apply unit through the service manager.
|
|
func (m *Machine) StartApply() error {
|
|
if m.Call == nil {
|
|
return fmt.Errorf("this process reaches no seat, so the apply was not started")
|
|
}
|
|
_, err := m.Call("seat:node-service-manager.start@"+m.Node, map[string]any{"scope": "system", "unit": ApplyUnit})
|
|
return err
|
|
}
|
|
|
|
// EndsAWait says whether an event of ZFS's ends a wait of the apply.
|
|
func EndsAWait(class string) bool {
|
|
return strings.HasSuffix(class, ".scrub_finish") || strings.HasSuffix(class, ".resilver_finish")
|
|
}
|
|
|
|
// ChangesAFinding says whether an event of ZFS's may change a device's state.
|
|
func ChangesAFinding(class string) bool {
|
|
return strings.HasSuffix(class, ".statechange") || strings.HasPrefix(class, "resource.fs.zfs.") ||
|
|
strings.HasSuffix(class, ".config_sync") || EndsAWait(class)
|
|
}
|
|
|
|
// DeclaredAdded says whether a line of `udevadm monitor` is the add of a block device a declared name links to.
|
|
// The declared names are read from the config file at each add, the links resolved now.
|
|
func (m *Machine) DeclaredAdded(line string) bool {
|
|
f := strings.Fields(line)
|
|
if len(f) < 4 || f[0] != "UDEV" || f[2] != "add" {
|
|
return false
|
|
}
|
|
added := filepath.Base(f[3])
|
|
text, err := m.ReadFile(ConfigFile)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
c, refused := ParseConfig(text, m.memMiB())
|
|
if len(refused) > 0 {
|
|
return false
|
|
}
|
|
for _, d := range append(m.resolveDeclared(c.CacheDevices, "cache"), m.resolveDeclared(c.LogDevices, "log")...) {
|
|
if d.Present && partOf(added, filepath.Base(d.Real)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// FollowUdev reads `udevadm monitor` and starts the apply on the add of a declared device.
|
|
func (m *Machine) FollowUdev(r io.Reader) error {
|
|
sc := bufio.NewScanner(r)
|
|
for sc.Scan() {
|
|
if m.DeclaredAdded(sc.Text()) {
|
|
if err := m.StartApply(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return sc.Err()
|
|
}
|
|
|
|
// StartUdev starts `udevadm monitor` and follows it until the context ends.
|
|
func (m *Machine) StartUdev(ctx context.Context) error {
|
|
cmd := exec.Command("udevadm", UdevArgv()...)
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
out, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := cmd.Start(); err != nil {
|
|
return err
|
|
}
|
|
go func() {
|
|
<-ctx.Done()
|
|
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
|
}()
|
|
ferr := m.FollowUdev(out)
|
|
_ = cmd.Wait()
|
|
return ferr
|
|
}
|
|
|
|
// FindingsOf are the findings `zpool status -P` shows now: `pool.<pool>.<device>` to state, for a pool or a device not
|
|
// ONLINE, the device by its own name, the last element of its path (A-341).
|
|
func FindingsOf(out string) (map[string]string, error) {
|
|
ps, err := ParseStatus(out)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
found := map[string]string{}
|
|
var walk func(pool string, d *Device)
|
|
walk = func(pool string, d *Device) {
|
|
for _, c := range d.Children {
|
|
if c.Kind != "section" && c.State != "" && c.State != "ONLINE" && c.State != "AVAIL" && c.State != "INUSE" {
|
|
found["pool."+pool+"."+filepath.Base(c.Name)] = c.State
|
|
}
|
|
walk(pool, c)
|
|
}
|
|
}
|
|
for _, p := range ps {
|
|
if p.State != "ONLINE" {
|
|
found["pool."+p.Name+".pool"] = p.State
|
|
}
|
|
if p.Root != nil {
|
|
walk(p.Name, p.Root)
|
|
}
|
|
}
|
|
return found, nil
|
|
}
|
|
|
|
// FindingsObserver is what the findings are written as: the module is their observer (A-32).
|
|
const FindingsObserver = "zfs"
|
|
|
|
// FindingKind is the one kind of the follower's findings, a pool or a device not ONLINE (A-341): each is its own key
|
|
// by its id, `zfs.<pool>.<device>` (`zfs.<pool>.pool` for the pool's own state), under the module's scope.
|
|
const FindingKind = "not-online"
|
|
|
|
// FindingsWriter is what the follower says its findings through (A-339): G's writer of the seat mesh-conditions, which
|
|
// keeps each in the state `findings` and says it on the module's finding stream.
|
|
type FindingsWriter interface {
|
|
Write(ctx context.Context, f seat.Finding) error
|
|
Clear(ctx context.Context, scope, id, kind, why string) error
|
|
}
|
|
|
|
// findingOf is the finding of one device or pool not ONLINE, by the name FindingsOf gives it, `pool.<pool>.<device>`
|
|
// (A-341): its id `zfs.<pool>.<device>`, its subject the module on the node.
|
|
func findingOf(name, state, node string, at time.Time) seat.Finding {
|
|
short := strings.TrimPrefix(name, "pool.")
|
|
return seat.Finding{Scope: meshconditionsclient.ScopeModule, ID: FindingsObserver + "." + short, Kind: FindingKind,
|
|
Observer: FindingsObserver, Subject: seat.Subject{Scope: meshconditionsclient.ScopeModule, ID: FindingsObserver, Machine: node},
|
|
Severity: meshconditionsclient.Warning, Source: FindingsObserver + "." + FindingKind,
|
|
Summary: fmt.Sprintf("%s is %s", short, state),
|
|
Headline: fmt.Sprintf("ZFS %s is %s", short, state),
|
|
Explanation: "ZFS reports this pool or device not ONLINE; zfs_pool_layout and zfs_pools say more. The finding clears when it is ONLINE again or leaves the pool.",
|
|
At: at.UTC()}
|
|
}
|
|
|
|
// ObserveFindings are the findings `zpool status -P` shows now, each as the writer says it (§27, G §384: judged anew
|
|
// from the module's own truth at its start and on each ask).
|
|
func (m *Machine) ObserveFindings(ctx context.Context) ([]seat.Finding, error) {
|
|
out, err := m.ask(ctx, "zpool", StatusPathsArgv()...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
now, err := FindingsOf(out)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var found []seat.Finding
|
|
for _, name := range sortedKeys(now) {
|
|
found = append(found, findingOf(name, now[name], m.Node, m.Now()))
|
|
}
|
|
return found, nil
|
|
}
|
|
|
|
// SyncFindings reads `zpool status -P` against the keys of the state `findings`, writes what is new or changed (the
|
|
// writer says only a change), and clears what is gone, each through the writer.
|
|
func (m *Machine) SyncFindings(ctx context.Context) error {
|
|
if m.States == nil || m.Findings == nil {
|
|
return fmt.Errorf("this process keeps no state, so the findings were not read")
|
|
}
|
|
found, err := m.ObserveFindings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
keys, err := m.States.Keys(StateFindings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
wanted := map[string]bool{}
|
|
for _, f := range found {
|
|
wanted[f.ID] = true
|
|
if err := m.Findings.Write(ctx, f); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
prefix := meshconditionsclient.ScopeModule + "." + FindingsObserver + "."
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
rest, mine := strings.CutPrefix(k, meshconditionsclient.ScopeModule+".")
|
|
id, kind := rest, ""
|
|
if i := strings.LastIndex(rest, "."); i > 0 {
|
|
id, kind = rest[:i], rest[i+1:]
|
|
}
|
|
if !mine || !strings.HasPrefix(k, prefix) || kind != FindingKind || wanted[id] {
|
|
continue
|
|
}
|
|
if err := m.Findings.Clear(ctx, meshconditionsclient.ScopeModule, id, FindingKind, "ONLINE again or left the pool"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StatusPathsArgv reads every pool's status with the paths its devices are held by.
|
|
func StatusPathsArgv() []string { return []string{"status", "-P"} }
|