jschoubben
879bc4bca2
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request
204 lines
8.2 KiB
Go
204 lines
8.2 KiB
Go
// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run:
|
|
//
|
|
// - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the
|
|
// Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now,
|
|
// restore beside the live data — and the health check, backup_health;
|
|
// - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by
|
|
// backup_health from what it leaves;
|
|
// - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233),
|
|
// judged by its unit.
|
|
//
|
|
// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module
|
|
// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the
|
|
// module's own tool only beside a check of another kind, so each runs as a process of its own.
|
|
//
|
|
// stdout is the MCP channel; everything this module says, it says on stderr.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Seat is the role this module holds.
|
|
const Seat = "node-backup"
|
|
|
|
func say(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...)
|
|
}
|
|
|
|
func main() {
|
|
where, err := whereFromEnv()
|
|
if err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
|
|
way := ""
|
|
if len(os.Args) > 1 {
|
|
way = os.Args[1]
|
|
}
|
|
switch way {
|
|
case "night":
|
|
nights(b)
|
|
case "measure":
|
|
measurements(b)
|
|
case "":
|
|
if err := stdio.Serve("", tools(b)); err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
default:
|
|
say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+
|
|
"measure it measures the data declared here", way)
|
|
os.Exit(2)
|
|
}
|
|
}
|
|
|
|
// nights runs at the hour, every module; and at start, if a night was missed — the machine was off
|
|
// or this module was not running at the hour — one soon rather than a day later. Not at once: a
|
|
// machine just started has its stores still coming up.
|
|
func nights(b *Backups) {
|
|
hour := 3
|
|
if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
|
|
hour = h
|
|
}
|
|
go counting(b)
|
|
if missedANight(b.Nights(), time.Now()) {
|
|
time.Sleep(10 * time.Minute)
|
|
night(b)
|
|
}
|
|
for {
|
|
time.Sleep(time.Until(nextNight(time.Now(), hour)))
|
|
night(b)
|
|
}
|
|
}
|
|
|
|
// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
|
|
// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
|
|
// each night (novox/hq ADR 0233).
|
|
func measurements(b *Backups) {
|
|
every := time.Hour
|
|
if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
|
every = d
|
|
}
|
|
time.Sleep(3 * time.Minute)
|
|
for {
|
|
if err := b.MeasureAll(context.Background(), false); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
time.Sleep(every)
|
|
}
|
|
}
|
|
|
|
func night(b *Backups) {
|
|
ctx := context.Background()
|
|
outcome, err := b.BackUp(ctx, "")
|
|
if err != nil {
|
|
say("the night did not run: %v", err)
|
|
return
|
|
}
|
|
var failed []string
|
|
for module, n := range outcome {
|
|
if !n.OK {
|
|
failed = append(failed, module)
|
|
}
|
|
}
|
|
if len(failed) > 0 {
|
|
say("the night left %s without a backup", strings.Join(failed, ", "))
|
|
}
|
|
if err := b.MeasureAll(ctx, true); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
// Sundays, the repository's own integrity with a sample of the data read back.
|
|
if time.Now().Weekday() == time.Sunday {
|
|
if err := b.Check(ctx); err != nil {
|
|
say("the repository does NOT check out: %v", err)
|
|
} else {
|
|
say("the repository checks out")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- the seat's verbs --------------------------------------------------------------------------
|
|
|
|
func str(description string) map[string]any {
|
|
return map[string]any{"type": "string", "description": description}
|
|
}
|
|
|
|
func arg(a map[string]any, k string) string {
|
|
v, _ := a[k].(string)
|
|
return strings.TrimSpace(v)
|
|
}
|
|
|
|
// verb is one of the seat's verbs: listed as `<seat>.<verb>`, so the runtime serves it on the seat's
|
|
// subject.
|
|
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
|
|
return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
|
|
}
|
|
|
|
func tools(b *Backups) []stdio.Tool {
|
|
return []stdio.Tool{
|
|
verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }),
|
|
verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Kept as taken by hand until a later good night of the module exists, and for keep-hand-taken-days in any case. Answers when it has started; `backed-up` says how it went.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) {
|
|
only := arg(a, "module")
|
|
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
|
go func() {
|
|
if _, err := b.BackUpNow(context.Background(), only); err != nil {
|
|
say("a backup asked for now failed: %v", err)
|
|
}
|
|
}()
|
|
started := only
|
|
if started == "" {
|
|
started = "every module on this machine"
|
|
}
|
|
return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil
|
|
}),
|
|
verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>, or in this module's state when a directory on the way belongs to another account. Swapping it in is a person's act.",
|
|
map[string]any{
|
|
"module": str("the module"),
|
|
"snapshot": str("the restore point (the newest when omitted)"),
|
|
"path": str("one of the module's directories (all of them when omitted)"),
|
|
},
|
|
func(a map[string]any) (any, error) {
|
|
module := arg(a, "module")
|
|
if module == "" {
|
|
return nil, fmt.Errorf("module is required")
|
|
}
|
|
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
|
|
}),
|
|
{Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " +
|
|
"(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " +
|
|
"did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " +
|
|
"to do; waits, naming the setting, when a retention setting is refused. (r)",
|
|
Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }},
|
|
// Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup
|
|
// delete` asks these. Not seat verbs — nobody else calls them.
|
|
{Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " +
|
|
"its module is gone from here — after taking a last restore point of it, tagged retired, so it can be " +
|
|
"restored until that restore point is forgotten. Refuses anything declared now. Answers at once; " +
|
|
"backup_deleted says how it went. Use the controller's `cleanup delete`, which records the hand act.",
|
|
Input: map[string]any{"module": str("the module it was"), "item": str("the item"), "path": str("where it is"),
|
|
"confirm": str("the item's name again"), "why": str("why — required"), "by": str("who decided"),
|
|
"via": str("mesh-controller when asked through its verbs")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
return b.DeleteRetired(context.Background(), arg(a, "module"), arg(a, "item"), arg(a, "path"),
|
|
arg(a, "confirm"), arg(a, "by"), arg(a, "why"))
|
|
}},
|
|
{Name: "backup_deleted", Description: "How the deletion of one retired item went: running, or done with " +
|
|
"its last restore point, or refused with why.",
|
|
Input: map[string]any{"path": str("where it is")},
|
|
Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
|
|
}
|
|
}
|