mesh-catalog/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/backup_test.go

package main

import (
	"strings"
	"testing"
)

// A failed backup before the send holds the delivery, and a release asks the backups again (novox/hq ADR 0404
// clause 4, issue 1068). The walk's backups field is the controller's contract.

func failedBackup() *WalkBackups {
	return &WalkBackups{State: "failed", Asked: []WalkBackupAsked{
		{Machine: "ace", Module: "mail", State: "done"},
		{Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}}
}

// aDeliveringWalk is a delivery whose walk was let go and runs, nothing sent yet.
func aDeliveringWalk(t *testing.T) (*world, string, string) {
	t.Helper()
	w := newWorld(t)
	id := IDOf("novox/app", head)
	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
	w.h.Checked(verdict("novox/app", 7, head, "pass"))
	w.h.PullMerged(merged("novox/app", 7, head, merge))
	w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true)))
	w.settleAll()
	if s := w.state(id); s != Delivering {
		t.Fatalf("setup: %s, want delivering", s)
	}
	return w, id, "plan-1"
}

func TestABackupFailedTakesADeliveringDeliveryToHeldOnlyWhileNothingWasSent(t *testing.T) {
	w, id, plan := aDeliveringWalk(t)
	sent := w.ctl.walk(plan)
	sent.Backups = failedBackup()
	sent.Modules["app"].SentAt = &w.now
	w.h.WalkMoved(w.ctl.put(sent))
	w.settleAll()
	if s := w.state(id); s == Held {
		t.Fatal("held for a failed backup after a module of the plan was sent")
	}
	w, id, plan = aDeliveringWalk(t)
	held := w.ctl.walk(plan)
	held.Backups = failedBackup()
	w.h.WalkMoved(w.ctl.put(held))
	w.settleAll()
	d := w.delivery(id)
	if d.State != Held {
		t.Fatalf("with a failed backup and nothing sent it is %s", d.State)
	}
	for _, want := range []string{"ace", "pg", "no space left on device"} {
		if !strings.Contains(d.HeldWhy, want) {
			t.Errorf("the held reason %q does not name %q", d.HeldWhy, want)
		}
	}
	// It does not lift by itself: the same facts read again leave it held.
	w.h.WalkMoved(w.ctl.put(w.ctl.walk(plan)))
	w.settleAll()
	if s := w.state(id); s != Held {
		t.Fatalf("read again it is %s, want it still held", s)
	}
}

func TestAReleaseOfADeliveryHeldForAFailedBackupAsksTheControllerAgainAndGoesToDelivering(t *testing.T) {
	w, id, plan := aDeliveringWalk(t)
	held := w.ctl.walk(plan)
	held.Backups = failedBackup()
	w.h.WalkMoved(w.ctl.put(held))
	w.settleAll()
	if s := w.state(id); s != Held {
		t.Fatalf("setup: %s, want held", s)
	}
	lettingGo := len(w.ctl.delivers)
	if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
		t.Fatal(err)
	}
	if got := w.ctl.backupsAgain; len(got) != 1 || got[0] != plan {
		t.Fatalf("the controller was asked to take the backups again %v, want once for %s", got, plan)
	}
	if len(w.ctl.delivers) != lettingGo {
		t.Errorf("a started walk was let go again: %v", w.ctl.delivers)
	}
	if s := w.state(id); s != Delivering {
		t.Fatalf("released, it is %s", s)
	}
	// The record the controller cleared is read: it stays delivering.
	w.h.WalkMoved(w.ctl.walk(plan))
	w.settleAll()
	if s := w.state(id); s != Delivering {
		t.Fatalf("with the backups cleared it is %s", s)
	}
	// Another failure, in a later record, holds it again.
	again := w.ctl.walk(plan)
	again.Backups = failedBackup()
	w.h.WalkMoved(w.ctl.put(again))
	w.settleAll()
	if s := w.state(id); s != Held {
		t.Fatalf("a failed backup in a later record left it %s", s)
	}
}

func TestARecordThatStillHoldsTheFailureAfterAReleaseDoesNotHoldItAgain(t *testing.T) {
	w, id, plan := aDeliveringWalk(t)
	held := w.ctl.walk(plan)
	held.Backups = failedBackup()
	stale := w.ctl.put(held)
	w.h.WalkMoved(stale)
	w.settleAll()
	if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
		t.Fatal(err)
	}
	w.h.WalkMoved(stale) // the record as it was before the controller cleared it
	w.settleAll()
	if s := w.state(id); s != Delivering {
		t.Fatalf("a record from before the release held it again: %s", s)
	}
}

func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) {
	// The controller's own path starts a walk with nobody's word: the held-to-delivering row for that must not
	// undo the backup hold.
	d := &Delivery{ID: "novox/app@c0", State: Held, MergedAs: "aaaa", Walk: &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 3,
		BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}}}
	if _, err := Apply(d, EvGo, false, Facts{}); err == nil {
		t.Fatal("a walk started on the controller's own path lifted the backup hold")
	}
}

func TestAReleaseWhoseBackupsTheControllerRefusedToAskAgainLeavesTheDeliveryHeld(t *testing.T) {
	w, id, plan := aDeliveringWalk(t)
	held := w.ctl.walk(plan)
	held.Backups = failedBackup()
	w.h.WalkMoved(w.ctl.put(held))
	w.settleAll()
	w.ctl.refuseBackupsAgain = true
	_, err := w.h.Release(id, "freed some space", "jochen")
	if err == nil || !strings.Contains(err.Error(), "held again") {
		t.Fatalf("the release of a refused word answered %v, want an error saying it is held again", err)
	}
	w.settleAll()
	d := w.delivery(id)
	if d.State != Held || d.BackupsAgainAfter != 0 || !strings.Contains(d.HeldWhy, "no space left on device") {
		t.Fatalf("after the refusal it is %s (again-after %d, why %q), want held for the backup", d.State, d.BackupsAgainAfter, d.HeldWhy)
	}
	// The controller takes the word the next time: the delivery goes on.
	w.ctl.refuseBackupsAgain = false
	if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
		t.Fatal(err)
	}
	if s := w.state(id); s != Delivering {
		t.Fatalf("released again, it is %s", s)
	}
}