package main // The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be // loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines // only — a typo in a setting let the repository grow until a disk condition fired. // // Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run // at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when // the night's process has stopped counting, when the last thinning failed, or when the retention file is // missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the // answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator // condition. The node-engine asks this through the node tools for the night's process; the controller // raises module.restic..unhealthy on the second look that says so, with these words. import ( "context" "fmt" "sort" "strings" "time" "git.novox.be/novox/mesh-sdk/go/health" ) // Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why, // waits), with what was found and what was said beside it. type Health struct { health.Answer Found []string `json:"found,omitempty"` Said []string `json:"said,omitempty"` } // Seen is what the health is judged from. type Seen struct { Declared []Declared DeclaredErr error Nights map[string]Night Good map[string]time.Time Run Run Retention InForce // Awake is the night's process's count of how long this machine was up since its newest good night; nil // before the process ever counted. Awake *Awake Now time.Time // Boot and Mono are this machine's boot and monotonic clock now (machineClock): within one boot, the count's // age is judged by the time the machine was awake (novox/hq issue 489). Boot string Mono time.Duration } func hours(d time.Duration) string { if d < 2*time.Hour { return fmt.Sprintf("%d minutes", int(d.Minutes())) } return fmt.Sprintf("%d hours", int(d.Hours())) } func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") } // Judge is the health of what was seen. Pure. func Judge(s Seen) Health { var found, said []string var waits []health.Wait if s.DeclaredErr != nil { found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+ "back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr)) } if s.Run.Failed != "" { found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+ "names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed)) } if a := s.Awake; a != nil { newest := "no night has backed anything up here yet" if !a.From.IsZero() { newest = "the newest good night was at " + clock(a.From) } switch { case a.age(s) > Stale: found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+ "night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+ "on this machine.", clock(a.Looked), hours(Stale), newest)) case a.Up > Overdue && len(s.Declared) > 0: // Nothing declared here, nothing to back up: no night is overdue (novox/hq issue 489). found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+ "night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+ "the repository, or never starts leaves no failed night behind; read the journal of restic-night "+ "(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up))) default: said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+ "said when that passes %s.", hours(a.Up), hours(Overdue))) } } failed := map[string]Night{} for _, d := range s.Declared { if n, ok := s.Nights[d.Module]; ok && !n.OK { failed[d.Module] = n } } names := make([]string, 0, len(failed)) for m := range failed { names = append(names, m) } sort.Strings(names) for _, m := range names { n := failed[m] since := "it has no restore point here" if g, ok := s.Good[m]; ok && !g.IsZero() { since = "its newest restore point is of " + clock(g) } found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+ `with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m)) } if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" { // Restore points taken by hand are let go before the counts' thinning, so they are gone when it fails after // (novox/hq issue 453): said, rather than "nothing was forgotten". kept := "Nothing was forgotten and every restore point is kept" if len(t.LetGo) > 0 { kept = "It let go " + strings.Join(t.LetGo, ", ") + ", taken by hand, past their days and a later night; " + "every other restore point is kept" } found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. %s, so the repository grows "+ "until a thinning succeeds; fix what the error names, then node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, "."), kept)) } r := s.Retention switch { case r.Refused == "" && r.Retention != nil: said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module, "+ "and each taken by hand until a later good night and %d days at least.", r.Daily, r.Weekly, r.Monthly, r.Hand)) case r.Setting != "": value := "not set" if r.Value != "" { value = fmt.Sprintf("%q", r.Value) } line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+ "nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+ "(mesh-controller.settings).", r.Setting, value, r.Setting) said = append(said, line) waits = append(waits, health.Wait{Part: "night", Setting: r.Setting, What: "a count from 1 to 9999 for " + r.Setting}) case strings.Contains(r.Refused, Unreadable): found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+ "read was written by an older build of this module, or changed on the machine, and the next push of this "+ "machine (mesh-controller.push) writes it readable again.", r.Refused)) case strings.HasSuffix(r.Refused, Missing): found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept until it is there again, rather than thinned by a number nobody set; push this machine "+ "(mesh-controller.push) so the mesh writes it again.", r.Refused)) case strings.Contains(r.Refused, RetentionFile+": "): // Read, and not in the form the mesh writes: something other than the mesh wrote it. found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept, so the repository grows. The mesh writes only daily=, weekly=, monthly= and hand=, so something else wrote "+ "this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) default: found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ "kept, so the repository grows. Fix what the error names.", r.Refused)) } if len(found) > 0 { why := strings.Join(found, " ") if len(waits) > 0 { why += " " + said[len(said)-1] } return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said} } if len(waits) > 0 { return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said} } return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said} } // healthySaid is what was seen, and only that, when nothing is wrong. func healthySaid(s Seen, said []string) string { var parts []string switch { case len(s.Declared) == 0: parts = append(parts, "No module here declares anything to back up.") case newestGood(s.Nights).IsZero(): parts = append(parts, "No good night is recorded here yet.") default: parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.", clock(newestGood(s.Nights)))) } switch t := s.Run.Thinned; { case t == nil: parts = append(parts, "No thinning is recorded yet.") case t.OK && t.Kept != nil: parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points%s.", clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly, letGoSaid(t.LetGo))) case t.Refused != "": parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At))) } return strings.Join(append(parts, said...), " ") } // Health reads what the nights left and the retention as it is now, and judges them. func (b *Backups) Health(ctx context.Context) Health { declared, err := b.Declared() boot, mono := machineClock() return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now(), Boot: boot, Mono: mono}) } // letGoSaid is the restore points taken by hand a thinning let go, as the health says them (novox/hq issue 453). func letGoSaid(ids []string) string { if len(ids) == 0 { return "" } return ", and let go " + strings.Join(ids, ", ") + ", taken by hand, past their days and a later night" }