package main import ( "context" "errors" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // itemBusFake records every call, in order, as the backup holder makes them. type itemBusFake struct { log []string restoring []string // what each read of the recovery module's restores answers, in turn unended []ItemRef runFailure error // waiting runs while the holder waits for a restore's restored; hold makes the wait last until its context ends. waiting func() hold bool } func (f *itemBusFake) Say(event string, ref ItemRef, body map[string]any) error { line := event + " " + ref.Module + "." + ref.Item if o, ok := body["outcome"]; ok { line += " " + fmt.Sprint(o) } if r, ok := body["restore"]; ok { line += " restore=" + fmt.Sprint(r) } if e, ok := body["error"]; ok { line += " error=" + fmt.Sprint(e) } if _, ok := body["at"]; !ok { line += " NO-AT" } f.log = append(f.log, line) return nil } func (f *itemBusFake) Unended() ([]ItemRef, error) { return f.unended, nil } func (f *itemBusFake) Restoring(module string) (string, error) { f.log = append(f.log, "read restores of "+module) if len(f.restoring) == 0 { return "", nil } act := f.restoring[0] f.restoring = f.restoring[1:] return act, nil } func (f *itemBusFake) WaitRestored(ctx context.Context, act string) error { f.log = append(f.log, "restored "+act) if f.waiting != nil { f.waiting() } if f.hold { <-ctx.Done() return ctx.Err() } return nil } // putOffFake is the state put-off: its keys with the time to live each was written with; expire plays the bus removing one. type putOffFake struct { keys map[string]PutOff ttl map[string]time.Duration expired map[string]func() } func newPutOffFake() *putOffFake { return &putOffFake{keys: map[string]PutOff{}, ttl: map[string]time.Duration{}, expired: map[string]func(){}} } func (f *putOffFake) Hold(key string, rec PutOff, ttl time.Duration, expired func()) (func(), error) { f.keys[key], f.ttl[key], f.expired[key] = rec, ttl, expired return func() { delete(f.keys, key); delete(f.expired, key) }, nil } func (f *putOffFake) All() (map[string]PutOff, error) { return f.keys, nil } func (f *putOffFake) expire(key string) { cb := f.expired[key] delete(f.keys, key) delete(f.expired, key) cb() } // findingsFake holds the findings written and not cleared. type findingsFake struct { held map[string]meshconditions.Finding } func (f *findingsFake) Write(_ context.Context, x meshconditions.Finding) error { f.held[x.Scope+"."+x.ID+"."+x.Kind] = x return nil } func (f *findingsFake) Clear(_ context.Context, scope, id, kind, _ string) error { delete(f.held, scope+"."+id+"."+kind) return nil } func (f *findingsFake) Held() []meshconditions.Finding { var out []meshconditions.Finding for _, x := range f.held { out = append(out, x) } return out } func itemsBackups(t *testing.T, bus *itemBusFake, broker string, runErr error) *Backups { t.Helper() return &Backups{Where: withData(t, composed, composedData), Node: "ace", Now: func() time.Time { return time.Date(2026, 11, 12, 3, 0, 0, 0, time.UTC) }, Say: quiet, Bus: bus, BrokerModule: func() (string, error) { return broker, nil }, Run: func(_ context.Context, name string, args ...string) (string, error) { if name == "restic" && len(args) > 5 && args[5] == "backup" { return "", runErr } return "[]", nil }} } func declaredOf(t *testing.T, b *Backups, module string) Declared { t.Helper() all, err := b.Declared() must(t, err) for _, d := range all { if d.Module == module { return d } } t.Fatalf("%s is not declared", module) return Declared{} } // Design 47 G §563 (BKP4): every item of a backed-up module says item-started, then item-ended with its outcome and `at`; a // cache item says nothing; a failed backup ends its items failed with why; the holder reads no restore for a module that does // not hold the seat mesh-broker. func TestEveryItemOfABackedUpModuleSaysItsStartAndItsEnd(t *testing.T) { bus := &itemBusFake{} b := itemsBackups(t, bus, "somebody-else", nil) if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK { t.Fatalf("%+v", n) } want := []string{"item-started postgres.store", "item-started postgres.dumps", "item-ended postgres.store done", "item-ended postgres.dumps done"} if !reflect.DeepEqual(bus.log, want) { t.Fatalf("said %q", bus.log) } bus.log = nil b = itemsBackups(t, bus, "somebody-else", errors.New("disk full")) if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); n.OK { t.Fatal("a failed backup was ok") } if len(bus.log) != 4 || bus.log[2] != "item-ended postgres.store failed error=disk full" { t.Fatalf("said %q", bus.log) } } // Design 47 G §563 (BKP5): for the holder of mesh-broker the holder says its start, then reads the recovery module's restores; // a restore that runs puts the items off naming its act, waits for that act's restored, starts again, and reads once more. func TestTheBackupOfTheBusGivesWayToARestoreThatRuns(t *testing.T) { bus := &itemBusFake{restoring: []string{"act-7"}} b := itemsBackups(t, bus, "postgres", nil) state := newPutOffFake() b.PutOffs, b.PutOffBound = state, 45*time.Minute // While it waits, `backed-up` lists the item put off with the restore's act and since when (BKP8, §302, §664). bus.waiting = func() { got, err := b.BackedUp(context.Background(), "postgres") must(t, err) if rec, ok := state.keys["postgres.store"]; !ok || rec.Restore != "act-7" || state.ttl["postgres.store"] != 45*time.Minute { t.Errorf("the put-off is not a key of the state put-off with the bound as its time to live: %+v %v", state.keys, state.ttl) } var store *ItemReport for i := range got[0].Data { if got[0].Data[i].ID == "store" { store = &got[0].Data[i] } } if store == nil || store.PutOff == nil || store.PutOff.Restore != "act-7" || !store.PutOff.Since.Equal(b.Now()) { t.Errorf("backed-up says the item as %+v", store) } } if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK { t.Fatalf("%+v", n) } want := []string{ "item-started postgres.store", "item-started postgres.dumps", "read restores of postgres", "item-ended postgres.store put-off restore=act-7", "item-ended postgres.dumps put-off restore=act-7", "restored act-7", "item-started postgres.store", "item-started postgres.dumps", "read restores of postgres", "item-ended postgres.store done", "item-ended postgres.dumps done"} if !reflect.DeepEqual(bus.log, want) { t.Fatalf("said %q", bus.log) } got, err := b.BackedUp(context.Background(), "postgres") must(t, err) for _, it := range got[0].Data { if it.PutOff != nil { t.Errorf("%s is still put off once it ran: %+v", it.ID, it.PutOff) } } } // Design 47 G §561, §563, A-155 (BKP6): a wait for a restore past put-off-bound, the key of the state put-off expiring, writes // the finding backup-waits for the item (scope the node), ends the wait (the backup fails, naming the bound) and holds the night's // lock no longer; the finding is cleared when the item starts again. func TestAWaitPastItsBoundWritesBackupWaitsAndEndsTheWait(t *testing.T) { bus := &itemBusFake{restoring: []string{"act-7"}, hold: true} b := itemsBackups(t, bus, "postgres", nil) state, findings := newPutOffFake(), &findingsFake{held: map[string]meshconditions.Finding{}} b.PutOffs, b.Findings = state, findings bus.waiting = func() { state.expire("postgres.dumps") } n := b.one(context.Background(), declaredOf(t, b, "postgres"), false) if n.OK || !strings.Contains(n.Error, "put-off-bound") || !strings.Contains(n.Error, "act-7") { t.Fatalf("the night %+v", n) } f, ok := findings.held["ace.postgres=2edumps.backup-waits"] if !ok || len(findings.held) != 1 || f.Scope != "ace" || f.Kind != FindingBackupWaits || !strings.Contains(f.Summary, "act-7") { t.Fatalf("the findings held: %+v", findings.held) } if len(state.keys) != 0 { t.Errorf("the keys left: %+v", state.keys) } // The next night: the item starts again and the finding is cleared. bus.restoring, bus.hold, bus.waiting = nil, false, nil if n := b.one(context.Background(), declaredOf(t, b, "postgres"), false); !n.OK || len(findings.held) != 0 { t.Fatalf("the night %+v, findings %+v", n, findings.held) } } // Design 47 G §561 (A-155): the setting put-off-bound is minutes in the file the manifest places; sixty when none is given. func TestPutOffBoundReadsItsSettingFile(t *testing.T) { file := filepath.Join(t.TempDir(), "put-off-bound") for text, want := range map[string]time.Duration{"15\n": 15 * time.Minute, "": DefaultPutOffBound} { must(t, os.WriteFile(file, []byte(text), 0o600)) got, err := putOffBoundFrom(func(string) string { return file }) if err != nil || got != want { t.Errorf("%q gave %v, %v", text, got, err) } } if got, _ := putOffBoundFrom(func(string) string { return "" }); got != DefaultPutOffBound { t.Errorf("no file gave %v", got) } must(t, os.WriteFile(file, []byte("soon"), 0o600)) if _, err := putOffBoundFrom(func(string) string { return file }); err == nil { t.Error("a bound that is no number was taken") } } // Design 47 G §563 (BKP4): at its start the holder ends every item it said started and never ended, failed with the error stopped. func TestItemsLeftStartedAreEndedFailedStoppedAtTheStart(t *testing.T) { bus := &itemBusFake{unended: []ItemRef{{"postgres", "store"}}} b := itemsBackups(t, bus, "postgres", nil) must(t, b.StopStale()) if !reflect.DeepEqual(bus.log, []string{"item-ended postgres.store failed error=stopped"}) { t.Fatalf("said %q", bus.log) } }