mesh-catalog/modules/restic/cmd/restic-backups/health.go

package main

// The module's health (novox/hq ADR 0240, issue 454), judged from what the nights left: a failure must be
// loud, and until this one a failed backup, a failed thinning and a refused retention were journal lines
// only — a typo in a setting let the repository grow until a disk condition fired.
//
// Unhealthy when a module declared here was NOT backed up at its last night, when the last night did not run
// at all, when the last thinning failed, or when the retention file is missing, cannot be read or is not in
// the form the mesh writes. A retention setting the night refuses is the operator's to give again: the
// answer is *waits*, naming the setting (ADR 0283), and the controller says it as the module's needs-operator
// condition. The node-engine asks this through the node tools for the night's process; the controller
// raises module.restic.<machine>.unhealthy on the second look that says so, with these words.

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
)

// Health is the answer the node-engine reads from the tool check: mesh-sdk's answer (state, healthy, why,
// waits), with what was found and what was said beside it.
type Health struct {
	health.Answer
	Found []string `json:"found,omitempty"`
	Said  []string `json:"said,omitempty"`
}

// Seen is what the health is judged from.
type Seen struct {
	Declared    []Declared
	DeclaredErr error
	Nights      map[string]Night
	Good        map[string]time.Time
	Run         Run
	Retention   InForce
}

func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }

// Judge is the health of what was seen. Pure.
func Judge(s Seen) Health {
	var found, said []string
	var waits []health.Wait
	if s.DeclaredErr != nil {
		found = append(found, fmt.Sprintf("What the modules here declare to back up does not read (%v), so no night can "+
			"back anything up; push this machine (mesh-controller.push) so the mesh writes it again.", s.DeclaredErr))
	}
	if s.Run.Failed != "" {
		found = append(found, fmt.Sprintf("The night of %s did not run: %s. No module was backed up; fix what the error "+
			"names, then take one with node-backup.now on this machine.", clock(s.Run.At), s.Run.Failed))
	}
	failed := map[string]Night{}
	for _, d := range s.Declared {
		if n, ok := s.Nights[d.Module]; ok && !n.OK {
			failed[d.Module] = n
		}
	}
	names := make([]string, 0, len(failed))
	for m := range failed {
		names = append(names, m)
	}
	sort.Strings(names)
	for _, m := range names {
		n := failed[m]
		since := "it has no restore point here"
		if g, ok := s.Good[m]; ok && !g.IsZero() {
			since = "its newest restore point is of " + clock(g)
		}
		found = append(found, fmt.Sprintf("%s was NOT backed up at %s: %s; %s. Fix what the error names, then take one "+
			`with node-backup.now {"module": "%s"} on this machine.`, m, clock(n.At), strings.TrimSuffix(n.Error, "."), since, m))
	}
	if t := s.Run.Thinned; t != nil && !t.OK && t.Error != "" {
		found = append(found, fmt.Sprintf("Thinning the restore points failed at %s: %s. Nothing was forgotten and every "+
			"restore point is kept, so the repository grows until a thinning succeeds; fix what the error names, then "+
			"node-backup.now on this machine thins again.", clock(t.At), strings.TrimSuffix(t.Error, ".")))
	}
	r := s.Retention
	switch {
	case r.Refused == "" && r.Retention != nil:
		said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.",
			r.Daily, r.Weekly, r.Monthly))
	case r.Setting != "":
		value := "not set"
		if r.Value != "" {
			value = fmt.Sprintf("%q", r.Value)
		}
		line := fmt.Sprintf("The retention setting %s is refused: it is %s, and it is a count from 1 to 9999. Until it is, "+
			"nothing is thinned and every restore point is kept, so the repository grows; set %s for restic on this machine "+
			"(mesh-controller.settings).", r.Setting, value, r.Setting)
		said = append(said, line)
		waits = append(waits, health.Wait{Part: "night", Setting: r.Setting,
			What: "a count from 1 to 9999 for " + r.Setting})
	case strings.HasSuffix(r.Refused, Missing):
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept until it is there again, rather than thinned by a number nobody set; push this machine "+
			"(mesh-controller.push) so the mesh writes it again.", r.Refused))
	default:
		found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+
			"kept, so the repository grows; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused))
	}
	if len(found) > 0 {
		why := strings.Join(found, " ")
		if len(waits) > 0 {
			why += " " + said[len(said)-1]
		}
		return Health{Answer: health.Unhealthy(why, waits...), Found: found, Said: said}
	}
	if len(waits) > 0 {
		return Health{Answer: health.Waiting(said[len(said)-1], waits...), Said: said}
	}
	why := "Every module declared here was backed up at its last night, and the last thinning succeeded."
	switch {
	case len(s.Declared) == 0:
		why = "No module here declares anything to back up."
	case len(s.Nights) == 0:
		why = "No night has run here yet."
	}
	if len(said) > 0 {
		why += " " + said[0]
	}
	return Health{Answer: health.Healthy(why), Said: said}
}

// Health reads what the nights left and the retention as it is now, and judges them.
func (b *Backups) Health(ctx context.Context) Health {
	declared, err := b.Declared()
	return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(),
		Retention: b.inForce(ctx)})
}