mesh-catalog/modules/restic/cmd/restic-backups/lock_test.go

package main

// The thinning waits for a repository another restic holds (novox/hq issue 620): met three times, each when a
// restic process outside the night's lock — a listing of the restore points while a delivery plan backs up a
// machine's data modules before their moves (ADR 0233) — held the repository's lock a few hundred milliseconds
// before the thinning asked for it, and the thinning failed at once.

import (
	"context"
	"errors"
	"slices"
	"strings"
	"testing"
	"time"
)

// lockedRestic is a fake restic whose repository another process holds for heldFor, as restic answers it: a
// forget without --retry-lock fails at once; one with it waits up to its bound, and succeeds when the lock is
// let go within it, else fails with restic's words. waited is how long the forgets it was asked waited.
type lockedRestic struct {
	handRestic
	heldFor time.Duration
	waited  []time.Duration
}

const lockSaid = "unable to create lock in backend: repository is already locked by PID 4242 on the-machine by root " +
	"(UID 0, GID 0) / lock was created at 2026-10-22 18:14:22 (246 ms ago)"

func (l *lockedRestic) run(ctx context.Context, name string, args ...string) (string, error) {
	if name != "restic" || args[5] != "forget" {
		return l.handRestic.run(ctx, name, args...)
	}
	l.calls = append(l.calls, "restic "+strings.Join(args[5:], " "))
	bound := time.Duration(0)
	if i := slices.Index(args, "--retry-lock"); i >= 0 && i+1 < len(args) {
		d, err := time.ParseDuration(args[i+1])
		if err != nil {
			return "", errors.New("invalid argument for --retry-lock: " + args[i+1])
		}
		bound = d
	}
	if l.heldFor > bound {
		l.waited = append(l.waited, bound)
		return "", errors.New(lockSaid)
	}
	l.waited = append(l.waited, l.heldFor)
	l.heldFor = 0
	return "", nil
}

func TestTheThinningWaitsForALockHeldBriefly(t *testing.T) {
	for _, held := range []time.Duration{3 * time.Second, 5 * time.Minute} {
		l := &lockedRestic{heldFor: held}
		b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: l.run, Now: time.Now, Say: quiet}
		_, err := b.BackUp(context.Background(), "")
		must(t, err)
		if th := b.LastRun().Thinned; th == nil || !th.OK {
			t.Errorf("held for %s: recorded %+v, want the thinning to wait for the lock and succeed", held, th)
		}
		if why := b.Health(context.Background()).Why; strings.Contains(why, "Thinning the restore points failed") {
			t.Errorf("held for %s: the health says %q", held, why)
		}
	}
}

func TestTheThinningStillFailsOnALockHeldPastItsBound(t *testing.T) {
	l := &lockedRestic{heldFor: thinningWaitsForALock + time.Minute}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: l.run, Now: time.Now, Say: quiet}
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	th := b.LastRun().Thinned
	if th == nil || th.OK || !strings.Contains(th.Error, "already locked") {
		t.Fatalf("recorded %+v, want the thinning failed on the lock", th)
	}
	if !slices.Equal(l.waited, []time.Duration{thinningWaitsForALock}) {
		t.Errorf("waited %v, want once, for %s", l.waited, thinningWaitsForALock)
	}
	why := b.Health(context.Background()).Why
	for _, want := range []string{"Thinning the restore points failed at", "already locked", "Nothing was forgotten",
		"node-backup.now on this machine thins again"} {
		if !strings.Contains(why, want) {
			t.Errorf("the health says %q, want %q in it", why, want)
		}
	}
}

// A restore point taken by hand that is let go by its id waits for the lock as well: it is the thinning too.
func TestTheThinningForgetsByIdWaitingForTheLock(t *testing.T) {
	now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
	l := &lockedRestic{heldFor: 3 * time.Second, handRestic: handRestic{made: "bbbbbbbb11", snaps: []Snapshot{
		{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}},
		{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}},
	}}}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: l.run, Now: func() time.Time { return now }, Say: quiet}
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	if th := b.LastRun().Thinned; th == nil || !th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) {
		t.Errorf("recorded %+v, want aaaaaaaa let go after waiting for the lock", th)
	}
}