mesh-controller/cmd/mesh-controller/held_back_integration_test.go

//go:build integration

package main

import (
	"bytes"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
)

func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asked := time.Now().Add(-time.Hour)
	aResolver(t, open, "c1c1c1c1c1", asked)
	for _, node := range []string{"anchor", "laptop"} {
		if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
			t.Fatal(err)
		}
	}
	// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
	if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
		t.Fatal(err)
	}
	compose := composeForPush(open)
	d := &recordedDelivery{inv: inv}
	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
		t.Fatal(err)
	}
	if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
		t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
	}
	digestOfLaptop := func() string {
		sent, err := inv.Outstanding(ctx, "laptop")
		if err != nil {
			t.Fatal(err)
		}
		return sent
	}
	before := digestOfLaptop()

	// The change merges; the policy is record. `push anchor` sends the anchor...
	aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
	d.declared = nil
	if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
		t.Fatal(err)
	}
	// ...and its cascade leaves the laptop, saying so.
	var said bytes.Buffer
	d.declared = nil
	refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
	if err != nil || len(refused) != 0 {
		t.Fatalf("the cascade failed: %v %v", refused, err)
	}
	if len(d.declared) != 0 {
		t.Fatalf("the cascade sent %v a build its policy records", d.declared)
	}
	if digestOfLaptop() != before {
		t.Fatal("the laptop's last send moved: it was sent the held build")
	}
	for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
		"upgrade policy records", "`push laptop` sends it"} {
		if !strings.Contains(said.String(), want) {
			t.Errorf("the push did not say %q:\n%s", want, said.String())
		}
	}

	// Held and owed something else at once — a peer joined: still not sent, and both said: why it
	// is held, and that what else it is owed waits with it.
	aThirdMachine(t, open)
	d.declared = nil
	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
		compose, d, "", nil, &said); err != nil {
		t.Fatal(err)
	}
	if len(d.declared) != 0 || digestOfLaptop() != before {
		t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
	}
	if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
		t.Fatalf("the push did not say both:\n%s", said.String())
	}

	// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
	// (novox/hq ADR 0236).
	if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
		t.Fatal(err)
	}
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
		compose, d, "", nil, &said); err != nil {
		t.Fatal(err)
	}
	if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
		t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
	}
	// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
	if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
		Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
		t.Fatal(err)
	}
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
		compose, d, "", nil, &said); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
		t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
	}
	if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
		t.Fatalf("the new send did not record the new build: %v", builds)
	}
}

func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
	if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
		t.Fatal(err)
	}
	compose := composeForPush(open)
	d := &recordedDelivery{inv: inv}
	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
		t.Fatal(err)
	}

	// `push spare`, the machine just placed: the others' peers change with it.
	aThirdMachine(t, open)
	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	var said bytes.Buffer
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
		t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
	}
	if strings.Contains(said.String(), "was not sent") {
		t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
	}

	// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
	record, err := inv.NodeByName(ctx, "laptop")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
		t.Fatal(err)
	}
	// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
	// question.
	if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
		t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
	}
}