mesh-controller/cmd/mesh-controller/ask_bus_moment_test.go

package main

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
)

// Issue 1334: a bus that does not answer for a moment (it reloads its user list, and JetStream's API has no responder for a few
// seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a
// module's own fault, or a bus silent for ten minutes, fails the plan.

func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
	p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}},
		Modules: map[string]*inventory.PlanModule{}}
	byName := map[string]inventory.Entry{}
	for _, m := range []string{"a", "b", "c"} {
		byName[m] = inventory.Entry{Source: inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main"}}
	}
	return p, byName
}

func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
	p, byName := aTierOfThree()
	bus := false
	was := askABuild
	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
		if source.Repository == "novox/b" && !bus {
			return "", fmt.Errorf("%w: the bucket mesh-controller_builds is not on the bus: nats: no responders available for request",
				inventory.ErrNoChain)
		}
		return "build-" + source.Repository, nil
	}
	t.Cleanup(func() { askABuild = was })

	for _, m := range []string{"a", "b", "c"} {
		askModule(t.Context(), p, m, byName)
	}
	if p.State != inventory.PlanBuilding {
		t.Fatalf("a bus that did not answer for a moment failed the plan: %s, %q", p.State, p.Note)
	}
	if p.Modules["a"].State != "asked" || p.Modules["c"].State != "asked" {
		t.Fatalf("the modules the bus did answer for were not asked: %+v %+v", p.Modules["a"], p.Modules["c"])
	}
	if s := p.Modules["b"]; s.State != "" {
		t.Fatalf("the module the bus did not answer for is %q, not left unasked", s.State)
	}
	if got := unaskedOf(*p); len(got) != 1 || got[0] != "b" {
		t.Fatalf("the next look would ask %v, not b", got)
	}
	if !strings.Contains(p.Note, "bus did not answer") {
		t.Fatalf("the plan says nothing of the wait: %q", p.Note)
	}

	// The next look, the bus answering: b is asked and the plan goes on.
	bus = true
	for _, m := range unaskedOf(*p) {
		askModule(t.Context(), p, m, byName)
	}
	if p.State != inventory.PlanBuilding || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
		t.Fatalf("b was not asked once the bus answered: %s %+v", p.State, p.Modules["b"])
	}
}

func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) {
	p, byName := aTierOfThree()
	was := askABuild
	askABuild = func(context.Context, buildSource, string, string) (string, error) {
		return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
	}
	t.Cleanup(func() { askABuild = was })
	askModule(t.Context(), p, "a", byName)
	if p.State != inventory.PlanBuilding {
		t.Fatalf("the first refusal failed the plan: %q", p.Note)
	}
	long := time.Now().Add(-busWaitBound - time.Minute)
	p.Modules["a"].BusWaitFrom = &long
	askModule(t.Context(), p, "a", byName)
	if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" {
		t.Fatalf("a bus silent past its bound did not fail the plan: %s %+v", p.State, p.Modules["a"])
	}
	// Said in plain words: the bus, the wait, and since when.
	want := "a could not be asked for: the bus did not answer an ask for 10m0s (since " + long.UTC().Format(time.RFC3339) +
		"): this kind of record lives on the bus"
	if !strings.HasPrefix(p.Note, want) || p.Modules["a"].Why != strings.TrimPrefix(want, "a could not be asked for: ")+p.Note[len(want):] {
		t.Fatalf("the plan says %q, not words starting %q", p.Note, want)
	}
	// The wait has ended: a Retry made while the bus still reloads waits again, and does not fail at its first refusal.
	if p.Modules["a"].BusWaitFrom != nil {
		t.Fatalf("a failed module still holds the start of its wait: %v", p.Modules["a"].BusWaitFrom)
	}
	p.State = inventory.PlanBuilding
	askModule(t.Context(), p, "a", byName)
	if p.State != inventory.PlanBuilding || p.Modules["a"].State != "" || p.Modules["a"].BusWaitFrom == nil {
		t.Fatalf("a retry made while the bus is silent did not wait afresh: %s %+v", p.State, p.Modules["a"])
	}
}

func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) {
	p, _ := aTierOfThree()
	if busWaiting(*p) {
		t.Fatal("a plan whose modules are not asked yet waits for the bus")
	}
	now := time.Now()
	p.Modules["b"] = &inventory.PlanModule{BusWaitFrom: &now}
	p.Modules["a"] = &inventory.PlanModule{State: "asked"}
	if got := busWaitingIn(*p); len(got) != 1 || got[0] != "b" {
		t.Fatalf("the modules waiting for the bus are %v, not b", got)
	}
	p.State = inventory.PlanFailed
	if busWaiting(*p) {
		t.Fatal("a plan that is not open is waited for")
	}
}

func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) {
	p, byName := aTierOfThree()
	delete(byName, "a")
	askModule(t.Context(), p, "a", byName)
	if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "no longer in the catalogue") {
		t.Fatalf("a module gone from the catalogue did not fail the plan: %s %q", p.State, p.Note)
	}

	p, byName = aTierOfThree()
	was := askABuild
	askABuild = func(context.Context, buildSource, string, string) (string, error) {
		return "", errors.New("git.example/novox/a is no repository the catalogue builds a module from")
	}
	t.Cleanup(func() { askABuild = was })
	askModule(t.Context(), p, "a", byName)
	if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" {
		t.Fatalf("a module's own refusal did not fail the plan: %s %+v", p.State, p.Modules["a"])
	}
}