mesh-controller/cmd/mesh-controller/rest_at_once_integration_test.go

//go:build integration

package main

import (
	"context"
	"slices"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
)

func aBacklogOnThree(t *testing.T) *backlogMesh {
	t.Helper()
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	if _, err := inv.AddNode(ctx, "desk"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
		t.Fatal(err)
	}
	wasHeard := releaseHeard
	releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
		return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
	}
	t.Cleanup(func() { releaseHeard = wasHeard })
	return b
}

func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklogOnThree(t)
	ctx := t.Context()
	said := events(t)
	gateEvery = 0
	for i := 0; i < 12; i++ {
		advancePlans(ctx, b.open)
	}
	p := b.release(t)
	if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
	}
	desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
	if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
		t.Fatalf("sends and judgings: %v", *said)
	}
	if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
		t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
	}
	if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
	}
}

func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklogOnThree(t)
	ctx := t.Context()
	inv := b.open.inventory
	gateEvery = 0
	b.broken["desk"] = true // desk fails what it is sent in the rest's step
	advancePlans(ctx, b.open)
	for i := 0; i < 12 && b.release(t).Open(); i++ {
		advancePlans(ctx, b.open)
	}
	p := b.release(t)
	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
	}
	if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
		t.Fatalf("laptop was never sent: %v", b.sent)
	}
	if !slices.Contains(p.Release.Done, "laptop") {
		t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
	}
	// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
	if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
	}
	// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
	if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
		t.Fatal("app's build is not marked failed")
	}
	if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
		t.Fatalf("desk was not put back: %v", sent)
	}
}

func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
	release := b.release(t)
	judging := release.Release.Gate
	if judging == nil || judging.Pending || judging.Passes == 0 {
		t.Fatalf("the release is not judging anchor: %+v", judging)
	}
	built := time.Now().UTC()
	plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
		Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
			Build: "build-app-c2"}}}
	if err := inv.SavePlan(ctx, &plan); err != nil {
		t.Fatal(err)
	}
	advancePlans(ctx, b.open)
	if len(b.sent) != 1 {
		t.Fatalf("anchor was sent again what it runs: %v", b.sent)
	}
	p, err := inv.PlanByID(ctx, "plan-app")
	if err != nil {
		t.Fatal(err)
	}
	g := p.Modules["app"].Gate
	if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
		t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
	}
	if p.Modules["app"].Previous != "c1" {
		t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
	}
}

func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	g := aGateMesh(t)
	ctx := t.Context()
	// The old controller sends anchor app c2 and is replaced before it keeps the send.
	made := sendRollout
	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
		if _, err := made(ctx, open, names); err != nil {
			return nil, err
		}
		return nil, context.Canceled
	}
	advancePlans(ctx, g.open)
	sendRollout = made
	if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
		t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
	}
	last, _, err := g.open.inventory.SentTo(ctx, "anchor")
	if err != nil {
		t.Fatal(err)
	}
	advancePlans(ctx, g.open)
	if len(g.sent) != 1 {
		t.Fatalf("the send was made again: %v", g.sent)
	}
	s := g.plan(t).Modules["app"]
	if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
		t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
	}
	// What anchor ran before was kept before the send, so a failure still puts it back (§5).
	if s.Previous != "c1" || s.Gate.From != "c1" {
		t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
	}
}

func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	p, err := releaseBacklog(ctx, b.open, "")
	if err != nil || p == nil {
		t.Fatalf("no release opened: %v", err)
	}
	if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
		t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
	}
	if err := inv.SavePlan(ctx, p); err != nil {
		t.Fatal(err)
	}
	// The send is made, and the controller is replaced before it keeps it.
	if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
		t.Fatal(err)
	}
	advancePlans(ctx, b.open)
	if len(b.sent) != 1 {
		t.Fatalf("anchor was sent again: %v", b.sent)
	}
	g := b.release(t).Release.Gate
	if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
		t.Fatalf("anchor is not judged on the send made: %+v", g)
	}
}

func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
	withTheGate(t)
	b := aBacklogOnThree(t)
	ctx := t.Context()
	gateEvery = 0
	b.broken["anchor"] = true
	advancePlans(ctx, b.open)
	for i := 0; i < 6 && b.release(t).Open(); i++ {
		advancePlans(ctx, b.open)
	}
	if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
	}
	for _, s := range b.sent {
		if !slices.Equal(s, []string{"anchor"}) {
			t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
		}
	}
}

func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklogOnThree(t)
	ctx := t.Context()
	register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
	if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
		t.Fatal(err)
	}
	said := events(t)
	gateEvery = 0
	for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
		advancePlans(ctx, b.open)
	}
	if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
		t.Fatalf("the order is %v", p.Release.Order)
	}
	laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
	if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
		t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
	}
}

func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
	withTheGate(t)
	judgeLeavesAsTheCore(t)
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	reporting := sendRollout
	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
		if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
			b.sent = append(b.sent, names)
			current, _ := open.inventory.CurrentBuilds(ctx)
			return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
				map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
		}
		return reporting(ctx, open, names)
	}
	t.Cleanup(func() { sendRollout = reporting })
	gateEvery = 0
	for i := 0; i < 8; i++ {
		advancePlans(ctx, b.open)
	}
	p := b.release(t)
	g := p.Release.Gates["laptop"]
	if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
		t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
	}
	since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
	g.Since = &since
	if err := inv.SavePlan(ctx, &p); err != nil {
		t.Fatal(err)
	}
	advancePlans(ctx, b.open)
	p = b.release(t)
	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
	}
	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
		t.Fatal("a silent machine's build was marked failed")
	}
	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
		t.Fatalf("laptop was put back: %v", sent)
	}
	obs, err := silentObservation(ctx, inv)
	if err != nil {
		t.Fatal(err)
	}
	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
		t.Fatalf("the silence is not said: %+v", obs)
	}
}