Merge pull request 'Back up every store: the restic module and the stores' contributions (hq ADR 0214, to-be 43)' (#49) from feat/node-backup into main

This commit was merged in pull request #49.
This commit is contained in:
2026-10-05 10:13:48 +00:00
15 changed files with 1003 additions and 6 deletions
+7
View File
@@ -222,5 +222,12 @@
"failregex": "^.*Failed authentication attempt for .* from <HOST>(?::\\d+)?\\s*$\n ^.*Invalid user .* from <HOST> port \\d+\\s*$\n ^.*User \\S+ from <HOST> not allowed because .*$",
"jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d"
}
],
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:data}\n"
}
]
}
+8 -1
View File
@@ -132,5 +132,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:data}\npath ${dir:config}\n"
}
]
}
+7
View File
@@ -558,5 +558,12 @@
"failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=<HOST>(?:,|$)",
"jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d"
}
],
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n"
}
]
}
+7
View File
@@ -79,5 +79,12 @@
"name": "mesh-vault",
"scope": "mesh"
}
],
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n"
}
]
}
+8 -1
View File
@@ -151,5 +151,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:data}\n"
}
]
}
+13 -1
View File
@@ -58,6 +58,11 @@
"type": "directory",
"mode": "0700"
},
{
"id": "dumps",
"type": "directory",
"mode": "0700"
},
{
"id": "net",
"type": "network",
@@ -113,5 +118,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n"
}
]
}
+22 -1
View File
@@ -65,6 +65,13 @@
"mode": "0700",
"owner": "10001:0"
},
{
"id": "dumps",
"type": "directory",
"path": "${dir:data}/backup",
"mode": "0700",
"owner": "10001:0"
},
{
"id": "net",
"type": "network",
@@ -86,6 +93,13 @@
"${dir:data}:/var/opt/mssql"
],
"secrets-in-environment": "the image documents only MSSQL_SA_PASSWORD, no _FILE and no configuration field; not convertible without a wrapper entrypoint"
},
{
"id": "backup-sql",
"type": "file",
"path": "${dir:state}/backup.sql",
"mode": "0600",
"content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n"
}
],
"build": {
@@ -112,5 +126,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n"
}
]
}
+8 -1
View File
@@ -117,5 +117,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "path ${dir:html}\n"
}
]
}
+15 -1
View File
@@ -74,6 +74,13 @@
"mode": "0700",
"owner": "999:70"
},
{
"id": "dumps",
"type": "directory",
"path": "${dir:store-data}/dumps",
"mode": "0700",
"owner": "999:70"
},
{
"id": "server",
"type": "container",
@@ -121,5 +128,12 @@
}
}
]
}
},
"contributions": [
{
"seat": "node-backup",
"kind": "backup",
"content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n"
}
]
}
@@ -0,0 +1,472 @@
// The machine's backups (novox/hq ADR 0214, to-be 43).
//
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
// directories already filled, into one file this module reads. Two kinds of line:
//
// run <shell command> run as root before the module's snapshot — a consistent dump of a store
// path <directory> a directory the module's snapshot keeps
//
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
//
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
// launching this binary runs as the operator's account, so restic and the run lines go through sudo
// without a prompt where the account is not root (ADR 0175 §4).
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"sync"
"time"
)
// Runner runs one command and answers what it printed, so the backups can be tested without restic
// or a store.
type Runner func(ctx context.Context, name string, args ...string) (string, error)
// escalated is the command as it is run: as given when this process is root, else through sudo
// without a prompt.
func escalated(uid int, name string, args []string) (string, []string) {
if uid == 0 {
return name, args
}
return "sudo", append([]string{"-n", name}, args...)
}
// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump
// that will not finish.
func execRunner(ctx context.Context, name string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 6*time.Hour)
defer cancel()
program, argv := escalated(os.Getuid(), name, args)
var stdout, stderr bytes.Buffer
cmd := exec.CommandContext(ctx, program, argv...)
cmd.Stdout, cmd.Stderr = &stdout, &stderr
if err := cmd.Run(); err != nil {
said := strings.TrimSpace(stderr.String())
if said == "" {
said = strings.TrimSpace(stdout.String())
}
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
}
lines := strings.Split(said, "\n")
if len(lines) > 3 {
lines = lines[len(lines)-3:]
}
if said == "" {
return "", fmt.Errorf("%s: %w", name, err)
}
return "", errors.New(strings.Join(lines, " / "))
}
return stdout.String(), nil
}
// Declared is what one module declared.
type Declared struct {
Module string `json:"module"`
Runs []string `json:"runs"`
Paths []string `json:"paths"`
}
var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`)
// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote
// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a
// line this holder does not read is refused, naming the module, rather than skipped.
func parseDeclared(text string) ([]Declared, error) {
var out []Declared
current := -1
for _, raw := range strings.Split(text, "\n") {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if m := moduleHeader.FindStringSubmatch(line); m != nil {
out = append(out, Declared{Module: m[1]})
current = len(out) - 1
continue
}
if strings.HasPrefix(line, "#") || current < 0 {
continue
}
kind, value, _ := strings.Cut(line, " ")
value = strings.TrimSpace(value)
d := &out[current]
switch {
case kind == "run" && value != "":
d.Runs = append(d.Runs, value)
case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"):
d.Paths = append(d.Paths, value)
default:
return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line)
}
}
kept := out[:0]
for _, d := range out {
if len(d.Runs) > 0 || len(d.Paths) > 0 {
kept = append(kept, d)
}
}
return kept, nil
}
// Snapshot is one restore point, as restic lists it.
type Snapshot struct {
ID string `json:"id"`
ShortID string `json:"short_id"`
Time time.Time `json:"time"`
Paths []string `json:"paths"`
Tags []string `json:"tags"`
Hostname string `json:"hostname"`
}
// Night is how one module's last night went.
type Night struct {
OK bool `json:"ok"`
At time.Time `json:"at"`
Snapshot string `json:"snapshot,omitempty"`
Error string `json:"error,omitempty"`
}
// Keep is the rotation (novox/hq ADR 0214).
var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6}
func tagOf(module string) string { return "module=" + module }
// Where is where the mesh put this module's things.
type Where struct {
Declared, Repository, PasswordFile, State string
}
func whereFromEnv() (Where, error) {
var missing []string
get := func(k string) string {
v := os.Getenv(k)
if v == "" {
missing = append(missing, k)
}
return v
}
w := Where{
Declared: get("MESH_BACKUP_DECLARED"),
Repository: get("MESH_BACKUP_REPOSITORY"),
PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"),
State: get("MESH_BACKUP_STATE"),
}
if len(missing) > 0 {
return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", "))
}
return w, nil
}
// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
// share a dump.
type Backups struct {
Where Where
Run Runner
Now func() time.Time
Say func(format string, args ...any)
mu sync.Mutex
}
func (b *Backups) restic(ctx context.Context, args ...string) (string, error) {
return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...)
}
// Declared is what the modules on this machine declared.
func (b *Backups) Declared() ([]Declared, error) {
raw, err := os.ReadFile(b.Where.Declared)
if err != nil {
return nil, err
}
return parseDeclared(string(raw))
}
func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") }
// Nights is how each module's last night went.
func (b *Backups) Nights() map[string]Night {
nights := map[string]Night{}
if raw, err := os.ReadFile(b.nightsFile()); err == nil {
_ = json.Unmarshal(raw, &nights)
}
return nights
}
func (b *Backups) record(module string, n Night) {
nights := b.Nights()
nights[module] = n
raw, _ := json.MarshalIndent(nights, "", " ")
if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil {
b.Say("recording %s's night failed: %v", module, err)
}
}
var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`)
// ensureRepository makes the repository the first time. One that exists and cannot be opened is
// said, never replaced: replacing it would discard every restore point to fix a password.
func (b *Backups) ensureRepository(ctx context.Context) error {
_, err := b.restic(ctx, "cat", "config")
if err == nil {
return nil
}
if !noRepository.MatchString(err.Error()) {
return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err)
}
b.Say("no repository at %s; making one", b.Where.Repository)
_, err = b.restic(ctx, "init")
return err
}
// one is one module's night: its run lines, then one snapshot of its paths. A failure is that
// module's alone.
func (b *Backups) one(ctx context.Context, d Declared) Night {
n := Night{At: b.Now().UTC()}
fail := func(err error) Night {
n.Error = err.Error()
b.Say("%s: NOT backed up: %s", d.Module, n.Error)
return n
}
for _, command := range d.Runs {
if _, err := b.Run(ctx, "sh", "-c", command); err != nil {
return fail(err)
}
}
if len(d.Paths) == 0 {
return fail(errors.New("it runs a dump and names no directory to keep it from"))
}
var missing []string
for _, p := range d.Paths {
if _, err := os.Stat(p); err != nil {
missing = append(missing, p)
}
}
if len(missing) > 0 {
return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", ")))
}
out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...)
if err != nil {
return fail(err)
}
scanner := bufio.NewScanner(strings.NewReader(out))
scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
for scanner.Scan() {
var m struct {
MessageType string `json:"message_type"`
SnapshotID string `json:"snapshot_id"`
}
if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 {
n.Snapshot = m.SnapshotID[:8]
}
}
n.OK = true
b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
return n
}
// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
b.mu.Lock()
defer b.mu.Unlock()
if err := b.ensureRepository(ctx); err != nil {
return nil, err
}
all, err := b.Declared()
if err != nil {
return nil, err
}
chosen := all
if only != "" {
chosen = nil
var names []string
for _, d := range all {
names = append(names, d.Module)
if d.Module == only {
chosen = append(chosen, d)
}
}
if len(chosen) == 0 {
return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
}
}
outcome := map[string]Night{}
for _, d := range chosen {
outcome[d.Module] = b.one(ctx, d)
b.record(d.Module, outcome[d.Module])
}
if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
"--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil {
b.Say("thinning the restore points failed, and every one is kept: %v", err)
}
return outcome, nil
}
func orNothing(names []string) string {
if len(names) == 0 {
return "nothing"
}
return strings.Join(names, ", ")
}
// Snapshots is the restore points, of one module or all.
func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) {
args := []string{"snapshots", "--json"}
if module != "" {
args = append(args, "--tag", tagOf(module))
}
out, err := b.restic(ctx, args...)
if err != nil {
return nil, err
}
var snaps []Snapshot
if strings.TrimSpace(out) == "" {
return nil, nil
}
if err := json.Unmarshal([]byte(out), &snaps); err != nil {
return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err)
}
return snaps, nil
}
// ModuleBackups is what `backed-up` says about one module.
type ModuleBackups struct {
Module string `json:"module"`
Runs int `json:"runs"`
Paths []string `json:"paths"`
LastNight *Night `json:"lastNight"`
RestorePoints int `json:"restorePoints"`
Newest *Snapshot `json:"newest,omitempty"`
}
// BackedUp is what is backed up here: each module, what it declared, its last night and its restore
// points.
func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) {
declared, err := b.Declared()
if err != nil {
return nil, err
}
nights := b.Nights()
snaps, _ := b.Snapshots(ctx, module)
out := []ModuleBackups{}
for _, d := range declared {
if module != "" && d.Module != module {
continue
}
m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths}
if n, ok := nights[d.Module]; ok {
m.LastNight = &n
}
for _, s := range snaps {
if slices.Contains(s.Tags, tagOf(d.Module)) {
m.RestorePoints++
newest := s
m.Newest = &newest
}
}
out = append(out, m)
}
return out, nil
}
// Restored is what a restore put where.
type Restored struct {
Module string `json:"module"`
From Snapshot `json:"from"`
Restored []string `json:"restored"`
Live string `json:"live"`
}
// Restore puts a module's data from a restore point BESIDE the live data: each directory as
// <path>.restored-<stamp>. A target that already exists is refused, never overwritten.
func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
b.mu.Lock()
defer b.mu.Unlock()
mine, err := b.Snapshots(ctx, module)
if err != nil {
return nil, err
}
if len(mine) == 0 {
return nil, fmt.Errorf("%s has no restore point on this machine", module)
}
chosen := mine[len(mine)-1]
if snapshot != "" {
found := false
var listed []string
for _, s := range mine {
listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339)))
if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) {
chosen, found = s, true
}
}
if !found {
return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", "))
}
}
paths := chosen.Paths
if path != "" {
if !slices.Contains(chosen.Paths, path) {
return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path)
}
paths = []string{path}
}
stamp := b.Now().UTC().Format("20060102-150405")
r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"}
for _, p := range paths {
target := p + ".restored-" + stamp
if _, err := os.Stat(target); err == nil {
return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target)
}
if _, err := b.restic(ctx, "restore", chosen.ID+":"+p, "--target", target); err != nil {
return nil, err
}
r.Restored = append(r.Restored, target)
b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target)
}
return r, nil
}
// Check is the weekly look at the repository's own integrity, with a sample of the data read back.
func (b *Backups) Check(ctx context.Context) error {
b.mu.Lock()
defer b.mu.Unlock()
_, err := b.restic(ctx, "check", "--read-data-subset", "5%")
return err
}
// nextNight is when the next night is due: the given hour, local time, today while it is still
// ahead, else tomorrow.
func nextNight(now time.Time, hour int) time.Time {
next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
if !next.After(now) {
next = next.AddDate(0, 0, 1)
}
return next
}
// missedANight is whether a night was missed: the newest good night of any module is older than a
// day and a bit — the machine was off, or this module was not running, at the hour.
func missedANight(nights map[string]Night, now time.Time) bool {
var newest time.Time
for _, n := range nights {
if n.OK && n.At.After(newest) {
newest = n.At
}
}
return newest.IsZero() || now.Sub(newest) > 26*time.Hour
}
@@ -0,0 +1,228 @@
package main
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
// restic is installed — over the real one, on throwaway directories.
import (
"context"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"
func placed(t *testing.T, declared string) Where {
t.Helper()
dir := t.TempDir()
must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600))
must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600))
return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"),
PasswordFile: filepath.Join(dir, "pw"), State: dir}
}
func must(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
func quiet(string, ...any) {}
func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) {
got, err := parseDeclared(composed)
must(t, err)
want := []Declared{
{Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}},
{Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("read %#v, want %#v", got, want)
}
}
func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) {
for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} {
if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") {
t.Errorf("%q: %v", bad, err)
}
}
}
func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) {
if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) {
t.Errorf("as an account: %s %v", p, a)
}
if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) {
t.Errorf("as root: %s %v", p, a)
}
}
func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) {
where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n")
var calls []string
run := func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
if name == "restic" {
line = "restic " + strings.Join(args[5:], " ")
}
calls = append(calls, line)
switch {
case line == "sh -c fail-it":
return "", errors.New("the dump failed")
case strings.HasPrefix(line, "restic backup"):
return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
}
return "", nil
}
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
outcome, err := b.BackUp(context.Background(), "")
must(t, err)
if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" {
t.Errorf("pg: %+v", outcome["pg"])
}
if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") {
t.Errorf("broken: %+v", outcome["broken"])
}
if !outcome["mail"].OK {
t.Errorf("mail failed with broken: %+v", outcome["mail"])
}
want := []string{
"restic cat config",
"sh -c dump-it",
"restic backup --json --tag module=pg /",
"sh -c fail-it",
"restic backup --json --tag module=mail /",
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
}
if !reflect.DeepEqual(calls, want) {
t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n"))
}
// Recorded, so `backed-up` and the missed-night check read it.
var nights map[string]Night
raw, err := os.ReadFile(filepath.Join(where.State, "nights.json"))
must(t, err)
must(t, json.Unmarshal(raw, &nights))
if nights["broken"].OK || !nights["mail"].OK {
t.Errorf("recorded %+v", nights)
}
}
func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) {
var calls []string
run := func(_ context.Context, _ string, args ...string) (string, error) {
calls = append(calls, strings.Join(args[5:], " "))
if args[5] == "cat" {
return "", errors.New("Fatal: wrong password or no key found")
}
return "", nil
}
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") {
t.Fatalf("got %v", err)
}
for _, c := range calls {
if c == "init" {
t.Fatal("it made a new repository over one it could not open")
}
}
}
func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) {
b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: func(context.Context, string, ...string) (string, error) { return "", nil },
Now: time.Now, Say: quiet}
outcome, err := b.BackUp(context.Background(), "")
must(t, err)
if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") {
t.Fatalf("pg: %+v", outcome["pg"])
}
}
func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) {
morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local)
if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) {
t.Errorf("from the morning: %v", got)
}
afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local)
if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) {
t.Errorf("from the afternoon: %v", got)
}
at := func(day int, ok bool) map[string]Night {
return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}}
}
for _, c := range []struct {
nights map[string]Night
missed bool
}{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} {
if got := missedANight(c.nights, afternoon); got != c.missed {
t.Errorf("%+v: missed %v", c.nights, got)
}
}
}
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) {
if _, err := exec.LookPath("restic"); err != nil {
t.Skip("restic is not installed")
}
root := t.TempDir()
store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps")
must(t, os.Mkdir(store, 0o700))
must(t, os.Mkdir(dumps, 0o700))
must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600))
where := placed(t, "# mail\npath "+store+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n")
// As whoever runs the test, against its own repository: no sudo.
run := func(ctx context.Context, name string, args ...string) (string, error) {
out, err := exec.CommandContext(ctx, name, args...).Output()
if ee, ok := err.(*exec.ExitError); ok {
return string(out), errors.New(string(ee.Stderr))
}
return string(out), err
}
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
ctx := context.Background()
night, err := b.BackUp(ctx, "")
must(t, err)
if !night["mail"].OK || !night["pg"].OK {
t.Fatalf("the night: %+v", night)
}
if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" {
t.Fatalf("the dump: %q", raw)
}
// The mistake.
must(t, os.Remove(filepath.Join(store, "mailbox")))
listed, err := b.BackedUp(ctx, "")
must(t, err)
if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 {
t.Fatalf("listed %+v", listed)
}
restored, err := b.Restore(ctx, "mail", "", "")
must(t, err)
if len(restored.Restored) != 1 || !strings.HasSuffix(restored.Restored[0], "store.restored-20261006-030000") {
t.Fatalf("restored %+v", restored)
}
if raw, _ := os.ReadFile(filepath.Join(restored.Restored[0], "mailbox")); string(raw) != "the only copy of a letter\n" {
t.Fatalf("the restored letter: %q", raw)
}
if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil {
t.Fatal("the restore wrote into the live directory")
}
// Never over anything: the same restore again finds its target taken.
if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") {
t.Fatalf("a second restore: %v", err)
}
}
+137
View File
@@ -0,0 +1,137 @@
// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the
// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It
// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the
// live data — and, beside them, runs the night that happens without anyone asking.
//
// stdout is the MCP channel; everything this module says, it says on stderr.
package main
import (
"context"
"fmt"
"os"
"strconv"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Seat is the role this module holds.
const Seat = "node-backup"
func say(format string, args ...any) {
fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...)
}
func main() {
where, err := whereFromEnv()
if err != nil {
say("%v", err)
os.Exit(1)
}
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
go nights(b)
if err := stdio.Serve("", tools(b)); err != nil {
say("%v", err)
os.Exit(1)
}
}
// nights runs at the hour, every module; and at start, if a night was missed — the machine was off
// or this module was not running at the hour — one soon rather than a day later. Not at once: a
// machine just started has its stores still coming up.
func nights(b *Backups) {
hour := 3
if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
hour = h
}
if missedANight(b.Nights(), time.Now()) {
time.Sleep(10 * time.Minute)
night(b)
}
for {
time.Sleep(time.Until(nextNight(time.Now(), hour)))
night(b)
}
}
func night(b *Backups) {
ctx := context.Background()
outcome, err := b.BackUp(ctx, "")
if err != nil {
say("the night did not run: %v", err)
return
}
var failed []string
for module, n := range outcome {
if !n.OK {
failed = append(failed, module)
}
}
if len(failed) > 0 {
say("the night left %s without a backup", strings.Join(failed, ", "))
}
// Sundays, the repository's own integrity with a sample of the data read back.
if time.Now().Weekday() == time.Sunday {
if err := b.Check(ctx); err != nil {
say("the repository does NOT check out: %v", err)
} else {
say("the repository checks out")
}
}
}
// ---- the seat's verbs --------------------------------------------------------------------------
func str(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
func arg(a map[string]any, k string) string {
v, _ := a[k].(string)
return strings.TrimSpace(v)
}
// verb is one of the seat's verbs: listed as `<seat>.<verb>`, so the runtime serves it on the seat's
// subject.
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
}
func tools(b *Backups) []stdio.Tool {
return []stdio.Tool{
verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
map[string]any{"module": str("one module (optional)")},
func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }),
verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
map[string]any{"module": str("one module (optional)")},
func(a map[string]any) (any, error) {
only := arg(a, "module")
// A night of a large store outlasts any call; it is started, and its outcome recorded.
go func() {
if _, err := b.BackUp(context.Background(), only); err != nil {
say("a backup asked for now failed: %v", err)
}
}()
started := only
if started == "" {
started = "every module on this machine"
}
return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil
}),
verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
map[string]any{
"module": str("the module"),
"snapshot": str("the restore point (the newest when omitted)"),
"path": str("one of the module's directories (all of them when omitted)"),
},
func(a map[string]any) (any, error) {
module := arg(a, "module")
if module == "" {
return nil, fmt.Errorf("module is required")
}
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
}),
}
}
+5
View File
@@ -0,0 +1,5 @@
module restic
go 1.25.0
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+64
View File
@@ -0,0 +1,64 @@
{
"module": "restic",
"version": "1",
"claims": [
{
"name": "node-backup",
"scope": "node",
"serves": [
"backed-up",
"now",
"restore"
]
}
],
"own-secrets": {
"repository": "${dir:state}/repository.secret"
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
},
{
"id": "repository",
"type": "directory",
"mode": "0700"
},
{
"id": "declared",
"type": "file",
"path": "${dir:state}/backups.conf",
"mode": "0600",
"content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}"
},
{
"id": "tool",
"type": "package",
"package": "restic"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/restic-backups",
"binary": "restic-backups",
"loads": [
"restic-backups"
],
"env": {
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}"
}
}
]
}
}