mesh-controller/cmd/mesh-controller/ask_bus_look_integration_test.go

//go:build integration

package main

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
)

// Issue 1334, the look: when the bus did not answer an ask of any module of a tier, no build is in flight and no event follows,
// so the look itself comes back (it fails, and the looker retries it), asks the modules again, and past the bound fails the plan.
func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	for _, name := range []string{"a", "b"} {
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"},
			inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
			t.Fatal(err)
		}
	}
	p := inventory.Plan{ID: "plan-bus", Repository: "novox/a", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
		State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
	if err := inv.SavePlan(ctx, &p); err != nil {
		t.Fatal(err)
	}
	bus := false
	was := askABuild
	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
		if !bus {
			return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
		}
		return "build-" + source.Repository, nil
	}
	t.Cleanup(func() { askABuild = was })
	look := func() bool {
		t.Helper()
		release, err := inv.HoldPlans(ctx, true)
		if err != nil {
			t.Fatal(err)
		}
		defer release()
		return advanceHeld(ctx, open)
	}
	read := func() inventory.Plan {
		t.Helper()
		got, err := inv.PlanByID(ctx, "plan-bus")
		if err != nil {
			t.Fatal(err)
		}
		return got
	}

	// Every module refused: the look says it failed, which has the looker come back; the plan is building and says why.
	if !look() {
		t.Fatal("a look that left the whole tier unasked for the bus did not fail, so nothing would come back to it")
	}
	if got := read(); got.State != inventory.PlanBuilding || !strings.Contains(got.Note, "bus did not answer") ||
		got.Modules["a"].State != "" || got.Modules["b"].State != "" {
		t.Fatalf("the plan after a refused tier: %s %q %+v %+v", got.State, got.Note, got.Modules["a"], got.Modules["b"])
	}
	// Again, still refused: the same fact is not saved again, and the look still comes back.
	before := read().Revision
	if !look() || read().Revision != before {
		t.Fatalf("a second refused look failed to come back or saved the plan again (revision %d, was %d)", read().Revision, before)
	}

	// The bus answers at the next look: both are asked, and nothing waits any more.
	bus = true
	if look() {
		t.Fatal("a look that asked the whole tier failed")
	}
	if got := read(); got.Modules["a"].State != "asked" || got.Modules["b"].State != "asked" || got.Modules["a"].BusWaitFrom != nil {
		t.Fatalf("the tier was not asked once the bus answered: %+v %+v", got.Modules["a"], got.Modules["b"])
	}
}

func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "a", Version: "1"},
		inventory.Source{Repository: "novox/a", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
		t.Fatal(err)
	}
	long := time.Now().UTC().Add(-busWaitBound - time.Minute)
	p := inventory.Plan{ID: "plan-bus", Repository: "novox/a", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &long}}}
	if err := inv.SavePlan(ctx, &p); err != nil {
		t.Fatal(err)
	}
	was := askABuild
	askABuild = func(context.Context, buildSource, string, string) (string, error) {
		return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
	}
	t.Cleanup(func() { askABuild = was })
	release, err := inv.HoldPlans(ctx, true)
	if err != nil {
		t.Fatal(err)
	}
	advanceHeld(ctx, open)
	release()
	got, err := inv.PlanByID(ctx, "plan-bus")
	if err != nil {
		t.Fatal(err)
	}
	if got.State != inventory.PlanFailed || !strings.Contains(got.Note, "the bus did not answer an ask for 10m0s (since ") {
		t.Fatalf("a bus silent past its bound did not fail the plan in words: %s %q", got.State, got.Note)
	}

	// Rebuilt by hand while the bus still reloads: said as waiting, not as a failure and not with an empty reason.
	joined, said, err := joinAPlan(ctx, open, "a")
	if err != nil || !joined || !strings.Contains(said, "not asked yet: the bus did not answer") {
		t.Fatalf("a rebuild left for the bus is said as %v %q %v", joined, said, err)
	}
	// And retried by hand: the module is named as waiting, not as asked again.
	got, _ = inv.PlanByID(ctx, "plan-bus")
	if got.State == inventory.PlanFailed {
		got.Modules["a"].State, got.Modules["a"].Why = "failed", "x"
		if err := inv.SavePlan(ctx, &got); err != nil {
			t.Fatal(err)
		}
		said, err := retryPlan(ctx, open, "plan-bus")
		if err != nil || !strings.Contains(said, "not asked yet: the bus did not answer") {
			t.Fatalf("a retry left for the bus is said as %q %v", said, err)
		}
	}
}