package main import ( "context" "fmt" "strings" "time" ) // The lines a pool's capacity is judged against: past CapacityWarn a warning, past CapacityFail // unhealthy. ZFS slows as a pool fills, and a full pool can refuse even a deletion. const ( CapacityWarn = 80 CapacityFail = 90 ) // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why, one // sentence per finding; warnings do not make it unhealthy. type Health struct { Healthy bool `json:"healthy"` Why string `json:"why"` Found []string `json:"found,omitempty"` Warnings []string `json:"warnings,omitempty"` } // Judge judges the pools from what zpool list and zpool status said. func Judge(pools []Pool, status []PoolStatus, now time.Time) Health { var found, warn []string if len(pools) == 0 { found = append(found, "ZFS is on this machine and no pool is imported.") } for _, p := range pools { s := statusOf(status, p.Name) state := p.Health if s != nil && s.State != "" { state = s.State } if state != "ONLINE" { line := fmt.Sprintf("Pool %s is %s", p.Name, state) if s != nil && s.Status != "" { line += ": " + strings.TrimSuffix(s.Status, ".") } if s != nil && s.Action != "" { line += ". Action: " + strings.TrimSuffix(s.Action, ".") } found = append(found, line+".") } switch { case p.Capacity > CapacityFail: found = append(found, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityFail, bytesSaid(p.Free))) case p.Capacity > CapacityWarn: warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityWarn, bytesSaid(p.Free))) } if s == nil { found = append(found, fmt.Sprintf("zpool status said nothing of pool %s.", p.Name)) continue } walk(s.Root, func(d *Device, _ int) { if d.Kind == "section" { return } if d.Errors() > 0 { found = append(found, fmt.Sprintf("%s %s in pool %s has errors: read %d, write %d, checksum %d.", d.Kind, d.Name, p.Name, d.Read, d.Write, d.Checksum)) } if d.Kind != "pool" && d.State != "" && d.State != "ONLINE" && d.State != "AVAIL" && d.State != "INUSE" { // A device not online already makes its pool DEGRADED or worse; said here by name. found = append(found, fmt.Sprintf("%s %s in pool %s is %s.", d.Kind, d.Name, p.Name, d.State)) } }, 0) if e := strings.TrimSpace(s.Errors); e != "" && e != "No known data errors" { found = append(found, fmt.Sprintf("Pool %s: %s.", p.Name, strings.TrimSuffix(e, "."))) } if ended := LastScrubEnded(s.Scan); ended != nil && now.Sub(*ended) > ScrubWarnAfter { warn = append(warn, fmt.Sprintf("Pool %s was last scrubbed %.0f days ago, more than %d.", p.Name, now.Sub(*ended).Hours()/24, int(ScrubWarnAfter.Hours()/24))) } else if s.Scan.State == "none" { warn = append(warn, fmt.Sprintf("Pool %s was never scrubbed.", p.Name)) } } if len(found) > 0 { return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn} } var parts []string for _, p := range pools { parts = append(parts, fmt.Sprintf("%s ONLINE, %d %% full", p.Name, p.Capacity)) } why := "Every pool is online with no errors: " + strings.Join(parts, "; ") + "." if len(warn) > 0 { why += " Warnings: " + strings.Join(warn, " ") } return Health{Healthy: true, Why: why, Warnings: warn} } // Health reads the pools and judges them. A machine without ZFS is unhealthy, loudly: the module was put // where it does not apply. func (m *Machine) Health(ctx context.Context) Health { pools, err := m.Pools(ctx) if err != nil { if a, ok := err.(errAbsent); ok { return Health{Healthy: false, Why: sentence(a.why) + " Unassign the module from this machine.", Found: []string{a.why}} } return Health{Healthy: false, Why: sentence("zpool list did not answer: " + err.Error()), Found: []string{err.Error()}} } status, err := m.Status(ctx) if err != nil { return Health{Healthy: false, Why: sentence("zpool status did not answer: " + err.Error()), Found: []string{err.Error()}} } return Judge(pools, status, m.Now()) } func sentence(s string) string { s = strings.TrimSpace(s) if s == "" { return s } s = strings.ToUpper(s[:1]) + s[1:] if !strings.HasSuffix(s, ".") { s += "." } return s }