mesh-catalog/modules/restic/cmd/restic-backups/putoff.go

package main

// An item put off for a restore (design 47 G §563, §302, §664, A-155): `backed-up` lists it with the restore's act and since
// when. The wait is a bound key `<module>.<item>` in the module's own state `put-off`, with a time to live of its own (the setting
// put-off-bound): the key is the record `backed-up` reads, its expiry is heard as the bus's own change, raises the finding
// `backup-waits` and ends the wait. No file and no timer keeps it.

import (
	"context"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
	"git.novox.be/novox/mesh-sdk/go/keys"
)

// PutOff says an item gave way to a restore: the act of that restore and since when.
type PutOff struct {
	Restore string    `json:"restore"`
	Since   time.Time `json:"since"`
}

// FindingBackupWaits is the backup holder's finding of an item put off past its bound (§561).
const FindingBackupWaits = "backup-waits"

// DefaultPutOffBound is the setting put-off-bound's default (§561, §557): sixty minutes.
const DefaultPutOffBound = 60 * time.Minute

// PutOffState is the module's own state `put-off`, whose keys live a time of their own.
type PutOffState interface {
	// Hold writes the key with its record and time to live, watched before it is written; expired is called once when the bus
	// removes the key at the end of its time. release removes the key and ends the watch, calling nothing.
	Hold(key string, rec PutOff, ttl time.Duration, expired func()) (release func(), err error)
	// All reads every key with its record.
	All() (map[string]PutOff, error)
}

// Findings is the backup holder's finding writer (mesh-sdk's held writer).
type Findings interface {
	Write(ctx context.Context, f meshconditions.Finding) error
	Clear(ctx context.Context, scope, id, kind, why string) error
	Held() []meshconditions.Finding
}

func putOffKey(ref ItemRef) string {
	return keys.EscapeName(ref.Module) + "." + keys.EscapeName(ref.Item)
}

func putOffRef(key string) (ItemRef, bool) {
	m, i, ok := strings.Cut(key, ".")
	if !ok {
		return ItemRef{}, false
	}
	module, e1 := keys.UnescapeName(m)
	item, e2 := keys.UnescapeName(i)
	return ItemRef{Module: module, Item: item}, e1 == nil && e2 == nil
}

func (b *Backups) putOffBound() time.Duration {
	if b.PutOffBound > 0 {
		return b.PutOffBound
	}
	return DefaultPutOffBound
}

// errPutOffExpired ends the wait for a restore past the bound.
var errPutOffExpired = errors.New("the restore did not end within the bound put-off-bound")

// keepPutOff records a module's items as put off for a restore, since now, each a bound key; expired is called once when the
// first of them is removed by the bus at the end of its time. It answers how to let the keys go.
func (b *Backups) keepPutOff(items []ItemRef, act string, expired func(ItemRef)) (func(), error) {
	if b.PutOffs == nil {
		return func() {}, nil
	}
	var releases []func()
	release := func() {
		for _, r := range releases {
			r()
		}
	}
	for _, ref := range items {
		ref := ref
		r, err := b.PutOffs.Hold(putOffKey(ref), PutOff{Restore: act, Since: b.Now().UTC()}, b.putOffBound(), func() { expired(ref) })
		if err != nil {
			release()
			return nil, fmt.Errorf("the put-off of %s.%s could not be kept: %w", ref.Module, ref.Item, err)
		}
		releases = append(releases, r)
	}
	return release, nil
}

// writeWaits says the finding backup-waits for an item put off past its bound, naming the restore's act (§561).
func (b *Backups) writeWaits(ref ItemRef, act string) {
	if b.Findings == nil {
		return
	}
	f := meshconditions.Finding{Scope: b.Node, ID: keys.EscapeName(ref.Module + "." + ref.Item), Kind: FindingBackupWaits,
		Subject:  meshconditions.Subject{Scope: "module", ID: "restic"},
		Severity: meshconditions.FindingSeverityWarning, Headline: "A backup waits on a restore",
		Summary:     fmt.Sprintf("the backup of %s.%s was put off for the restore %s of the recovery module and that restore has not ended within %s", ref.Module, ref.Item, act, b.putOffBound()),
		Explanation: fmt.Sprintf("the item %s.%s gave way to the restore %s at its start and waits for its restored", ref.Module, ref.Item, act),
		Needs:       "the restore ended or cancelled in the recovery module (recovery_cancel); the backup then starts again at the next night",
		Source:      "backups", Observer: "restic", At: b.Now().UTC()}
	if err := b.Findings.Write(context.Background(), f); err != nil {
		b.Say("the finding %s of %s.%s could not be written: %v", FindingBackupWaits, ref.Module, ref.Item, err)
	}
}

// clearWaits clears the finding backup-waits of the items that started again.
func (b *Backups) clearWaits(items []ItemRef) {
	if b.Findings == nil {
		return
	}
	held := map[string]bool{}
	for _, f := range b.Findings.Held() {
		if f.Kind == FindingBackupWaits {
			held[f.Scope+"\x00"+f.ID] = true
		}
	}
	for _, ref := range items {
		id := keys.EscapeName(ref.Module + "." + ref.Item)
		if !held[b.Node+"\x00"+id] {
			continue
		}
		if err := b.Findings.Clear(context.Background(), b.Node, id, FindingBackupWaits, "the item started again"); err != nil {
			b.Say("the finding %s of %s.%s could not be cleared: %v", FindingBackupWaits, ref.Module, ref.Item, err)
		}
	}
}

// withPutOff says on a module's item reports which items are put off, read from the state's keys.
func (b *Backups) withPutOff(module string, reports []ItemReport) []ItemReport {
	if b.PutOffs == nil {
		return reports
	}
	all, err := b.PutOffs.All()
	if err != nil {
		b.Say("the items put off could not be read: %v", err)
		return reports
	}
	for key, rec := range all {
		ref, ok := putOffRef(key)
		if !ok || ref.Module != module {
			continue
		}
		for i := range reports {
			if reports[i].ID == ref.Item {
				put := rec
				reports[i].PutOff = &put
			}
		}
	}
	return reports
}

// PutOffBoundEnv names the file of the setting put-off-bound, in minutes: the time the wait for a restore may last before
// backup-waits is written and the wait ends (§561, §557).
const PutOffBoundEnv = "MESH_BACKUP_PUT_OFF_BOUND_FILE"

// putOffBoundFrom reads the setting from its file; it is the default (sixty minutes) when the process is given no file or an
// empty one, and refused when it does not read as a number of minutes above zero.
func putOffBoundFrom(getenv func(string) string) (time.Duration, error) {
	path := strings.TrimSpace(getenv(PutOffBoundEnv))
	if path == "" {
		return DefaultPutOffBound, nil
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		return 0, fmt.Errorf("the setting put-off-bound cannot be read: %w", err)
	}
	text := strings.TrimSpace(string(raw))
	if text == "" {
		return DefaultPutOffBound, nil
	}
	minutes, err := strconv.ParseFloat(text, 64)
	if err != nil || minutes <= 0 {
		return 0, fmt.Errorf("the setting put-off-bound is %q, not a number of minutes above zero", text)
	}
	return time.Duration(minutes * float64(time.Minute)), nil
}