jschoubben
cbe9fa7a2c
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
A hung night, a lock held for hours or a lock that cannot be taken left no failed night behind, so it stayed healthy. The night's process now counts how long the machine was up since its newest good night, by the monotonic clock so a sleeping laptop is not blamed, and past 26 hours it is raised. The retention file is written 0644 and root's instead of read through sudo: the account reads it and still cannot change it. An unreadable file names its cause, and the healthy answer says only what was seen.
204 lines
8.1 KiB
Go
204 lines
8.1 KiB
Go
// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, three ways to run:
|
|
//
|
|
// - with no argument, launched by the machine's tool runtime and speaking MCP to it over stdio through the
|
|
// Go SDK (ADR 0193, ADR 0198): the node-backup seat's three verbs — what is backed up, take one now,
|
|
// restore beside the live data — and the health check, backup_health;
|
|
// - `night`, a process the node-engine runs: the night that happens without anyone asking, judged by
|
|
// backup_health from what it leaves;
|
|
// - `measure`, a process the node-engine runs: the hourly measuring of the data declared here (ADR 0233),
|
|
// judged by its unit.
|
|
//
|
|
// The night and the measuring were goroutines of the tool runtime's process until novox/hq issue 454: a module
|
|
// raises a condition only through the health of something it runs (ADR 0240, ADR 0256), and a check by the
|
|
// module's own tool only beside a check of another kind, so each runs as a process of its own.
|
|
//
|
|
// stdout is the MCP channel; everything this module says, it says on stderr.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Seat is the role this module holds.
|
|
const Seat = "node-backup"
|
|
|
|
func say(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...)
|
|
}
|
|
|
|
func main() {
|
|
where, err := whereFromEnv()
|
|
if err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
|
|
way := ""
|
|
if len(os.Args) > 1 {
|
|
way = os.Args[1]
|
|
}
|
|
switch way {
|
|
case "night":
|
|
nights(b)
|
|
case "measure":
|
|
measurements(b)
|
|
case "":
|
|
if err := stdio.Serve("", tools(b)); err != nil {
|
|
say("%v", err)
|
|
os.Exit(1)
|
|
}
|
|
default:
|
|
say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+
|
|
"measure it measures the data declared here", way)
|
|
os.Exit(2)
|
|
}
|
|
}
|
|
|
|
// nights runs at the hour, every module; and at start, if a night was missed — the machine was off
|
|
// or this module was not running at the hour — one soon rather than a day later. Not at once: a
|
|
// machine just started has its stores still coming up.
|
|
func nights(b *Backups) {
|
|
hour := 3
|
|
if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
|
|
hour = h
|
|
}
|
|
go counting(b)
|
|
if missedANight(b.Nights(), time.Now()) {
|
|
time.Sleep(10 * time.Minute)
|
|
night(b)
|
|
}
|
|
for {
|
|
time.Sleep(time.Until(nextNight(time.Now(), hour)))
|
|
night(b)
|
|
}
|
|
}
|
|
|
|
// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
|
|
// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
|
|
// each night (novox/hq ADR 0233).
|
|
func measurements(b *Backups) {
|
|
every := time.Hour
|
|
if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
|
every = d
|
|
}
|
|
time.Sleep(3 * time.Minute)
|
|
for {
|
|
if err := b.MeasureAll(context.Background(), false); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
time.Sleep(every)
|
|
}
|
|
}
|
|
|
|
func night(b *Backups) {
|
|
ctx := context.Background()
|
|
outcome, err := b.BackUp(ctx, "")
|
|
if err != nil {
|
|
say("the night did not run: %v", err)
|
|
return
|
|
}
|
|
var failed []string
|
|
for module, n := range outcome {
|
|
if !n.OK {
|
|
failed = append(failed, module)
|
|
}
|
|
}
|
|
if len(failed) > 0 {
|
|
say("the night left %s without a backup", strings.Join(failed, ", "))
|
|
}
|
|
if err := b.MeasureAll(ctx, true); err != nil {
|
|
say("measuring the data declared here failed: %v", err)
|
|
}
|
|
// Sundays, the repository's own integrity with a sample of the data read back.
|
|
if time.Now().Weekday() == time.Sunday {
|
|
if err := b.Check(ctx); err != nil {
|
|
say("the repository does NOT check out: %v", err)
|
|
} else {
|
|
say("the repository checks out")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- the seat's verbs --------------------------------------------------------------------------
|
|
|
|
func str(description string) map[string]any {
|
|
return map[string]any{"type": "string", "description": description}
|
|
}
|
|
|
|
func arg(a map[string]any, k string) string {
|
|
v, _ := a[k].(string)
|
|
return strings.TrimSpace(v)
|
|
}
|
|
|
|
// verb is one of the seat's verbs: listed as `<seat>.<verb>`, so the runtime serves it on the seat's
|
|
// subject.
|
|
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
|
|
return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
|
|
}
|
|
|
|
func tools(b *Backups) []stdio.Tool {
|
|
return []stdio.Tool{
|
|
verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }),
|
|
verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
|
map[string]any{"module": str("one module (optional)")},
|
|
func(a map[string]any) (any, error) {
|
|
only := arg(a, "module")
|
|
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
|
go func() {
|
|
if _, err := b.BackUp(context.Background(), only); err != nil {
|
|
say("a backup asked for now failed: %v", err)
|
|
}
|
|
}()
|
|
started := only
|
|
if started == "" {
|
|
started = "every module on this machine"
|
|
}
|
|
return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil
|
|
}),
|
|
verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>, or in this module's state when a directory on the way belongs to another account. Swapping it in is a person's act.",
|
|
map[string]any{
|
|
"module": str("the module"),
|
|
"snapshot": str("the restore point (the newest when omitted)"),
|
|
"path": str("one of the module's directories (all of them when omitted)"),
|
|
},
|
|
func(a map[string]any) (any, error) {
|
|
module := arg(a, "module")
|
|
if module == "" {
|
|
return nil, fmt.Errorf("module is required")
|
|
}
|
|
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
|
|
}),
|
|
{Name: "backup_health", Description: "Whether this machine's backups hold, as the module's health check " +
|
|
"(novox/hq ADR 0240, issue 454): unhealthy when a module was NOT backed up at its last night, the last night " +
|
|
"did not run, the last thinning failed, or the retention file is missing or unreadable — each said with what " +
|
|
"to do; waits, naming the setting, when a retention setting is refused. (r)",
|
|
Run: func(map[string]any) (any, error) { return b.Health(context.Background()), nil }},
|
|
// Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup
|
|
// delete` asks these. Not seat verbs — nobody else calls them.
|
|
{Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " +
|
|
"its module is gone from here — after taking a last restore point of it, tagged retired, so it can be " +
|
|
"restored until that restore point is forgotten. Refuses anything declared now. Answers at once; " +
|
|
"backup_deleted says how it went. Use the controller's `cleanup delete`, which records the hand act.",
|
|
Input: map[string]any{"module": str("the module it was"), "item": str("the item"), "path": str("where it is"),
|
|
"confirm": str("the item's name again"), "why": str("why — required"), "by": str("who decided"),
|
|
"via": str("mesh-controller when asked through its verbs")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
return b.DeleteRetired(context.Background(), arg(a, "module"), arg(a, "item"), arg(a, "path"),
|
|
arg(a, "confirm"), arg(a, "by"), arg(a, "why"))
|
|
}},
|
|
{Name: "backup_deleted", Description: "How the deletion of one retired item went: running, or done with " +
|
|
"its last restore point, or refused with why.",
|
|
Input: map[string]any{"path": str("where it is")},
|
|
Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
|
|
}
|
|
}
|