Walks run at most daily and stop after ten minutes or two million files; datasets are read from their counters and large items from their top level only. The database platform's tables are dumped with pg_dumpall rather than copied as live files.
176 lines
6.6 KiB
Go
176 lines
6.6 KiB
Go
// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the
|
|
// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It
|
|
// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the
|
|
// live data — and, beside them, runs the night that happens without anyone asking.
|
|
//
|
|
// stdout is the MCP channel; everything this module says, it says on stderr.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Seat is the role this module holds.
|
|
const Seat = "node-backup"
|
|
|
|
func say(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...)
|
|
}
|
|
|
|
func main() {
|
|
where, err := whereFromEnv()
|
|
if err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
|
|
go nights(b)
|
|
go measurements(b)
|
|
if err := stdio.Serve("", tools(b)); err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
// nights runs at the hour, every module; and at start, if a night was missed — the machine was off
|
|
// or this module was not running at the hour — one soon rather than a day later. Not at once: a
|
|
// machine just started has its stores still coming up.
|
|
func nights(b *Backups) {
|
|
hour := 3
|
|
if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
|
|
hour = h
|
|
}
|
|
if missedANight(b.Nights(), time.Now()) {
|
|
time.Sleep(10 * time.Minute)
|
|
night(b)
|
|
}
|
|
for {
|
|
time.Sleep(time.Until(nextNight(time.Now(), hour)))
|
|
night(b)
|
|
}
|
|
}
|
|
|
|
// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
|
|
// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
|
|
// each night (novox/hq ADR 0233).
|
|
func measurements(b *Backups) {
|
|
every := time.Hour
|
|
if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
|
every = d
|
|
}
|
|
time.Sleep(3 * time.Minute)
|
|
for {
|
|
if err := b.MeasureAll(context.Background(), false); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
time.Sleep(every)
|
|
}
|
|
}
|
|
|
|
func night(b *Backups) {
|
|
ctx := context.Background()
|
|
outcome, err := b.BackUp(ctx, "")
|
|
if err != nil {
|
|
say("the night did not run: %v", err)
|
|
return
|
|
}
|
|
var failed []string
|
|
for module, n := range outcome {
|
|
if !n.OK {
|
|
failed = append(failed, module)
|
|
}
|
|
}
|
|
if len(failed) > 0 {
|
|
say("the night left %s without a backup", strings.Join(failed, ", "))
|
|
}
|
|
if err := b.MeasureAll(ctx, true); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
// Sundays, the repository's own integrity with a sample of the data read back.
|
|
if time.Now().Weekday() == time.Sunday {
|
|
if err := b.Check(ctx); err != nil {
|
|
say("the repository does NOT check out: %v", err)
|
|
} else {
|
|
say("the repository checks out")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- the seat's verbs --------------------------------------------------------------------------
|
|
|
|
func str(description string) map[string]any {
|
|
return map[string]any{"type": "string", "description": description}
|
|
}
|
|
|
|
func arg(a map[string]any, k string) string {
|
|
v, _ := a[k].(string)
|
|
return strings.TrimSpace(v)
|
|
}
|
|
|
|
// verb is one of the seat's verbs: listed as `<seat>.<verb>`, so the runtime serves it on the seat's
|
|
// subject.
|
|
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
|
|
return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
|
|
}
|
|
|
|
func tools(b *Backups) []stdio.Tool {
|
|
return []stdio.Tool{
|
|
verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }),
|
|
verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) {
|
|
only := arg(a, "module")
|
|
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
|
go func() {
|
|
if _, err := b.BackUp(context.Background(), only); err != nil {
|
|
say("a backup asked for now failed: %v", err)
|
|
}
|
|
}()
|
|
started := only
|
|
if started == "" {
|
|
started = "every module on this machine"
|
|
}
|
|
return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil
|
|
}),
|
|
verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
|
|
map[string]any{
|
|
"module": str("the module"),
|
|
"snapshot": str("the restore point (the newest when omitted)"),
|
|
"path": str("one of the module's directories (all of them when omitted)"),
|
|
},
|
|
func(a map[string]any) (any, error) {
|
|
module := arg(a, "module")
|
|
if module == "" {
|
|
return nil, fmt.Errorf("module is required")
|
|
}
|
|
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
|
|
}),
|
|
// Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup
|
|
// delete` asks these. Not seat verbs — nobody else calls them.
|
|
{Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " +
|
|
"its module is gone from here — after taking a last restore point of it, tagged retired, so it can be " +
|
|
"restored until that restore point is forgotten. Refuses anything declared now. Answers at once; " +
|
|
"backup_deleted says how it went. Use the controller's `cleanup delete`, which records the hand act.",
|
|
Input: map[string]any{"module": str("the module it was"), "item": str("the item"), "path": str("where it is"),
|
|
"confirm": str("the item's name again"), "why": str("why — required"), "by": str("who decided"),
|
|
"via": str("mesh-controller when asked through its verbs")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
return b.DeleteRetired(context.Background(), arg(a, "module"), arg(a, "item"), arg(a, "path"),
|
|
arg(a, "confirm"), arg(a, "by"), arg(a, "why"))
|
|
}},
|
|
{Name: "backup_deleted", Description: "How the deletion of one retired item went: running, or done with " +
|
|
"its last restore point, or refused with why.",
|
|
Input: map[string]any{"path": str("where it is")},
|
|
Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
|
|
}
|
|
}
|