mesh-catalog/modules/restic/cmd/restic-backups/health.go

package main

// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be
// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines
// only — a typo in a setting let the repository grow until a disk condition fired.
//
// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run
// at all, when no night has backed anything up while this machine was up for more than a day (awake.go), when
// the night's process has stopped counting, when the last thinning failed, or when the retention file is
// missing, cannot be read or is not in the form the mesh writes. A retention setting the night refuses is the operator's to give again: the
// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator
// condition. The node-engine asks this through the node tools for the night's process; the controller
// raises module.restic.<machine>.unhealthy on the second look that says so, with these words.

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
)

// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why,
// waits), with what was found and what was said beside it.
type Health struct {
	health.Answer
	Found []string `json:"found,omitempty"`
	Said  []string `json:"said,omitempty"`
}

// Seen is what the health is judged from.
type Seen struct {
	Declared    []Declared
	DeclaredErr error
	Nights      map[string]Night
	Good        map[string]time.Time
	Run         Run
	Retention   InForce
	// Awake is the night's process's count of how long this machine was up since its newest good night; nil
	// before the process ever counted.
	Awake *Awake
	Now   time.Time
}

func hours(d time.Duration) string {
	if d < 2*time.Hour {
		return fmt.Sprintf("%d minutes", int(d.Minutes()))
	}
	return fmt.Sprintf("%d hours", int(d.Hours()))
}

func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }

// Judge is the health of what was seen. Pure.
func Judge(s Seen) Health {
	var found, said []string
	var waits []health.Wait
	if s.DeclaredErr != nil {
		found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+
			"back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr))
	}
	if s.Run.Failed != "" {
		found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+
			"names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed))
	}
	if a := s.Awake; a != nil {
		newest := "no night has backed anything up here yet"
		if !a.From.IsZero() {
			newest = "the newest good night was at " + clock(a.From)
		}
		switch {
		case s.Now.Sub(a.Looked) > Stale:
			found = append(found, fmt.Sprintf("The night's process last counted at %s, more than %s ago, so whether a "+
				"night is overdue is not known; %s. Read the journal of restic-night (node-service-manager.journal) "+
				"on this machine.", clock(a.Looked), hours(Stale), newest))
		case a.Up > Overdue:
			found = append(found, fmt.Sprintf("No night is happening: %s, and this machine has been up with the "+
				"night's process running for %s since. A night that hangs, waits on a restore or a deletion holding "+
				"the repository, or never starts leaves no failed night behind; read the journal of restic-night "+
				"(node-service-manager.journal), then take one with node-backup.now on this machine.", newest, hours(a.Up)))
		default:
			said = append(said, fmt.Sprintf("Up %s with the night's process running since the newest good night; "+
				"said when that passes %s.", hours(a.Up), hours(Overdue)))
		}
	}
	failed := map[string]Night{}
	for _, d := range s.Declared {
		if n, ok := s.Nights[d.Module]; ok && !n.OK {
			failed[d.Module] = n
		}
	}
	names := make([]string, 0, len(failed))
	for m := range failed {
		names = append(names, m)
	}
	sort.Strings(names)
	for _, m := range names {
		n := failed[m]
		since := "it has no restore point here"
		if g, ok := s.Good[m]; ok && !g.IsZero() {
			since = "its newest restore point is of " + clock(g)
		}
		found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+
			`with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m))
	}
	if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" {
		// Restore points taken by hand are let go before the counts' thinning, so they are gone when it fails after
		// (novox/hq issue 453): said, rather than "nothing was forgotten".
		kept := "Nothing was forgotten and every restore point is kept"
		if len(t.LetGo) > 0 {
			kept = "It let go " + strings.Join(t.LetGo, ", ") + ", taken by hand, past their days and a later night; " +
				"every other restore point is kept"
		}
		found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. %s, so the repository grows "+
			"until a thinning succeeds; fix what the error names, then node-backup.now on this machine thins again.",
			clock(t.At), strings.TrimSuffix(t.Error, "."), kept))
	}
	r := s.Retention
	switch {
	case r.Refused == "" && r.Retention != nil:
		said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module, "+
			"and each taken by hand until a later good night and %d days at least.", r.Daily, r.Weekly, r.Monthly, r.Hand))
	case r.Setting != "":
		value := "not set"
		if r.Value != "" {
			value = fmt.Sprintf("%q", r.Value)
		}
		line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+
			"nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+
			"(mesh-controller.settings).", r.Setting, value, r.Setting)
		said = append(said, line)
		waits = append(waits, health.Wait{Part: "night", Setting: r.Setting,
			What: "a count from 1 to 9999 for " + r.Setting})
	case strings.Contains(r.Refused, Unreadable):
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept, so the repository grows. The mesh writes the file readable (0644) and root's; one the account cannot "+
			"read was written by an older build of this module, or changed on the machine, and the next push of this "+
			"machine (mesh-controller.push) writes it readable again.", r.Refused))
	case strings.HasSuffix(r.Refused, Missing):
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept until it is there again, rather than thinned by a number nobody set; push this machine "+
			"(mesh-controller.push) so the mesh writes it again.", r.Refused))
	case strings.Contains(r.Refused, RetentionFile+": "):
		// Read, and not in the form the mesh writes: something other than the mesh wrote it.
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept, so the repository grows. The mesh writes only daily=, weekly=, monthly= and hand=, so something else wrote "+
			"this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused))
	default:
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept, so the repository grows. Fix what the error names.", r.Refused))
	}
	if len(found) > 0 {
		why := strings.Join(found, " ")
		if len(waits) > 0 {
			why += " " + said[len(said)-1]
		}
		return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said}
	}
	if len(waits) > 0 {
		return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said}
	}
	return Health{Answer: health.Healthy(healthySaid(s, said)), Said: said}
}

// healthySaid is what was seen, and only that, when nothing is wrong.
func healthySaid(s Seen, said []string) string {
	var parts []string
	switch {
	case len(s.Declared) == 0:
		parts = append(parts, "No module here declares anything to back up.")
	case newestGood(s.Nights).IsZero():
		parts = append(parts, "No good night is recorded here yet.")
	default:
		parts = append(parts, fmt.Sprintf("No module declared here failed its last night; the newest good night was at %s.",
			clock(newestGood(s.Nights))))
	}
	switch t := s.Run.Thinned; {
	case t == nil:
		parts = append(parts, "No thinning is recorded yet.")
	case t.OK && t.Kept != nil:
		parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points%s.",
			clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly, letGoSaid(t.LetGo)))
	case t.Refused != "":
		parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At)))
	}
	return strings.Join(append(parts, said...), " ")
}

// Health reads what the nights left and the retention as it is now, and judges them.
func (b *Backups) Health(ctx context.Context) Health {
	declared, err := b.Declared()
	return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(),
		Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()})
}

// letGoSaid is the restore points taken by hand a thinning let go, as the health says them (novox/hq issue 453).
func letGoSaid(ids []string) string {
	if len(ids) == 0 {
		return ""
	}
	return ", and let go " + strings.Join(ids, ", ") + ", taken by hand, past their days and a later night"
}