mesh-controller/cmd/mesh-controller/plan_backups_hold_test.go

package main

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
)

// A backup that failed or stayed silent holds the plan, is said, and a verb asks it again (novox/hq ADR 0404
// decision 4, issue 1066).

// aHeldPlan is a plan whose backup on ace failed and whose backup on g14 is done.
func aHeldPlan(t *testing.T) (*inventory.Plan, backupFacts) {
	t.Helper()
	entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}
	facts := backupFactsFrom(entries, holders("ace", "g14"))
	seat := &aFakeBackupSeat{nights: map[string]*night{}}
	seat.install(t)
	p := aPlanOf("mail")
	stepBackups(t.Context(), p, facts, backupT0)
	seat.nights["ace/mail"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"}
	seat.nights["g14/mail"] = &night{OK: true, At: backupT0.Add(time.Minute)}
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed || err != nil {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	return p, facts
}

// The held plan says which machine, which module and the backup's own words in its note, and is not called tried again.
func TestAHeldPlanSaysTheMachineTheModuleAndTheWordsInItsNote(t *testing.T) {
	p, _ := aHeldPlan(t)
	for _, want := range []string{"ace", "mail", "no space left on device", "delivery-backups-again"} {
		if !strings.Contains(p.Note, want) {
			t.Fatalf("note %q lacks %q", p.Note, want)
		}
	}
	if strings.Contains(p.Note, "tried again") || p.State != inventory.PlanBuilding {
		t.Fatalf("a held plan is not failed and not retried: %q %s", p.Note, p.State)
	}
	if !waitingOnBackups(*p) {
		t.Fatal("a plan held by its backups is no tier late (S3 leaves it out)")
	}
}

// delivery-backups-again clears the failed records, and the next step asks those backups again and only those.
func TestBackupsAgainClearsTheFailedRecordsAndTheNextStepAsksThemAgain(t *testing.T) {
	p, facts := aHeldPlan(t)
	again, err := askBackupsAgain(p, "mesh-delivery", "a person released it")
	if err != nil || len(again) != 1 || again[0].Machine != "ace" || again[0].Module != "mail" {
		t.Fatalf("again=%+v err=%v", again, err)
	}
	if p.Backups.State != inventory.BackupPending {
		t.Fatalf("state %q", p.Backups.State)
	}
	seat := &aFakeBackupSeat{nights: map[string]*night{}}
	seat.install(t)
	now := backupT0.Add(10 * time.Minute)
	if proceed, err := stepBackups(t.Context(), p, facts, now); proceed || err != nil {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	if strings.Join(seat.asked, " ") != "ace/mail" {
		t.Fatalf("asked %v; only the failed backup is asked again", seat.asked)
	}
	seat.nights["ace/mail"] = &night{OK: true, At: now.Add(time.Minute)}
	if proceed, _ := stepBackups(t.Context(), p, facts, now.Add(2*time.Minute)); !proceed {
		t.Fatal("the plan must go on once the backup it asked again is done")
	}
	if strings.Contains(p.Note, "delivery-backups-again") {
		t.Fatalf("the hold's note stays after it ended: %q", p.Note)
	}
}

// Refused for a plan with no failed backup.
func TestBackupsAgainIsRefusedForAPlanWithNoFailedBackup(t *testing.T) {
	if _, err := askBackupsAgain(aPlanOf("mail"), "x", "y"); err == nil {
		t.Fatal("a plan with no backups was asked again")
	}
	p := aPlanOf("mail")
	p.Backups = &inventory.PlanBackups{State: inventory.BackupPending,
		Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}}
	if _, err := askBackupsAgain(p, "x", "y"); err == nil {
		t.Fatal("a plan whose backups are pending was asked again")
	}
}

// The verb's argument vector: the plan and a reason are required.
func TestTheVerbDeliveryBackupsAgainTakesAPlanAndAWhy(t *testing.T) {
	got, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"})
	want := []string{"delivery", "backups-again", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}
	if err != nil || strings.Join(got, "|") != strings.Join(want, "|") {
		t.Fatalf("got %v %v, want %v", got, err, want)
	}
	if _, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1"}); err == nil {
		t.Fatal("no reason was accepted")
	}
}

// One condition per held plan, the backup one, with the words in its evidence; none for a plan only waiting.
func TestAHeldPlanRaisesOneConditionWithTheBackupsWords(t *testing.T) {
	held, _ := aHeldPlan(t)
	waiting := aPlanOf("mail")
	waiting.ID = "plan-w"
	waiting.Backups = &inventory.PlanBackups{State: inventory.BackupPending,
		Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}}
	got := observeBackupHolds([]inventory.Plan{*held, *waiting})
	if len(got) != 1 {
		t.Fatalf("observations %+v", got)
	}
	o := got[0]
	if o.Kind != kindBackupHeld || o.Scope != conditions.ScopePlan || o.ID != held.ID ||
		!strings.Contains(o.Said, "no space left on device") || !strings.Contains(o.Said, "mail") || !strings.Contains(o.Said, "ace") {
		t.Fatalf("%+v", o)
	}
}

// A holder off the bus cannot be asked: the plan waits within the bound, retried by the look and not by the error
// path, and at the bound the backup is failed with the reason, so the hold and the condition take over.
func TestABackupHolderThatCannotBeAskedIsFailedAtTheBoundWithTheReason(t *testing.T) {
	seat := &aFakeBackupSeat{}
	seat.install(t)
	was := askBackupNow
	t.Cleanup(func() { askBackupNow = was })
	calls := 0
	askBackupNow = func(_ context.Context, machine, module string) error {
		calls++
		return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: no responders", machine)}
	}
	entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace"}, anItem("a", ""))}
	facts := backupFactsFrom(entries, holders("ace"))
	p := aPlanOf("mail")
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0); proceed || err != nil || p.Backups.State != inventory.BackupPending {
		t.Fatalf("proceed=%v err=%v %+v", proceed, err, p.Backups)
	}
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin/2)); proceed || err != nil ||
		p.Backups.State != inventory.BackupPending || calls != 2 || p.Backups.Asked[0].AskedAt != nil {
		t.Fatalf("within the bound: proceed=%v err=%v calls=%d %+v", proceed, err, calls, p.Backups)
	}
	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin)); proceed || err != nil {
		t.Fatalf("proceed=%v err=%v", proceed, err)
	}
	a := p.Backups.Asked[0]
	if p.Backups.State != inventory.BackupFailed || a.State != inventory.BackupFailed ||
		!strings.Contains(a.Words, "could not be asked: ") || !strings.Contains(a.Words, "no responders") {
		t.Fatalf("%+v", p.Backups)
	}
	if strings.Contains(p.Note, "tried again") || !strings.Contains(p.Note, "held") {
		t.Fatalf("note %q", p.Note)
	}
}