mesh-controller/cmd/mesh-controller/held_back_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/overlay"
)

// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
// else that moved is still a consequence the push sends (ADR 0083).
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
	current := map[string]inventory.CurrentBuild{
		"resolver": {Commit: "c2c2c2c2c2"},
		"agent":    {Commit: "a2", RollOut: true},
		"network":  {},
	}
	modules := []string{"network", "resolver", "agent"}
	sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}

	// A module whose policy records moved: held, naming it, both builds and why.
	why := heldBack("laptop", modules, sent, true, current, nil)
	if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
		!strings.Contains(why[0], "upgrade policy records") {
		t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
	}

	// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
	sent["resolver"] = "c2c2c2c2c2"
	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
		t.Fatalf("a machine whose builds are all current was held: %v", why)
	}

	// A module whose policy rolls out moved, and no plan holds it: sent, as before.
	sent["agent"] = "a1"
	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
		t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
	}

	// The last send's builds are not known: held whole.
	if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
		!strings.Contains(why[0], "not known") {
		t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
	}

	// A module the machine was never sent, under a recording policy: held, and said so.
	delete(sent, "resolver")
	sent["agent"] = "a2"
	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
		!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
		t.Fatalf("a module never sent under a recording policy: %v", why)
	}
}

// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
// naming some other machine must not send them for it.
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
	at := time.Now()
	current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
	sent := map[string]string{"agent": "a1"}
	waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
		"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}

	why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
	if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
		t.Fatalf("a machine the plan has not reached was not held: %v", why)
	}
	// The first machine itself was sent by the plan: not held by it.
	if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
		t.Fatalf("the plan's first machine was held: %v", why)
	}
	// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
	for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
		plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
			Modules: map[string]*inventory.PlanModule{"agent": s}}}
		if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
			t.Errorf("%s: not held: %v", what, why)
		}
	}
	// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
	for what, plans := range map[string][]inventory.Plan{
		"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
			"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
		"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
			"agent": {State: "failed"}}}},
		"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
			"agent": {State: "built"}}}},
	} {
		if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
			t.Errorf("%s: held: %v", what, why)
		}
	}
}

// A send records the build of each module it carried; a module left out of it keeps the build it
// was last sent, since the machine keeps that one.
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
	current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
	got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
		current, map[string]string{"a": "a1", "b": "b1"})
	if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
		t.Fatalf("carried %v, wanted %v", got, want)
	}
	// Not known before: the left-out module is not recorded at all, so it reads as never sent.
	got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
	if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
		t.Fatalf("carried %v, wanted %v", got, want)
	}
}

// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
// reads the machine as current — and writes down which machines it declared.
type recordedDelivery struct {
	inv      *inventory.Inventory
	declared []string
}

func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }

func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
	r.declared = append(r.declared, s.node)
	return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
}

// aResolver is a module built from a repository, at a commit, with something on the machine that
// says which build it is.
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
	t.Helper()
	m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
		{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
	}}
	if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
		Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
		t.Fatal(err)
	}
}

// A third machine on the private network: once it is sent, every other machine's peers change with
// it, which is a consequence a push must still send — no build moved.
func aThirdMachine(t *testing.T, open *stores) {
	t.Helper()
	ctx := t.Context()
	record, err := open.inventory.AddNode(ctx, "spare")
	if err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
		t.Fatal(err)
	}
	reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
		{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
		{"name": "systemd", "present": true}}})
	if err != nil {
		t.Fatal(err)
	}
	var profile map[string]any
	if err := json.Unmarshal(reported, &profile); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
		t.Fatal(err)
	}
	if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
		t.Fatal(err)
	}
	if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
		t.Fatal(err)
	}
}

// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asked := time.Now().Add(-time.Hour)
	aResolver(t, open, "c1c1c1c1c1", asked)
	for _, node := range []string{"anchor", "laptop"} {
		if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
			t.Fatal(err)
		}
	}
	gens, err := generators(ctx, open)
	if err != nil {
		t.Fatal(err)
	}
	compose := composeForPush(open, gens)
	d := &recordedDelivery{inv: inv}
	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
		t.Fatal(err)
	}
	if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
		t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
	}
	digestOfLaptop := func() string {
		sent, err := inv.Outstanding(ctx, "laptop")
		if err != nil {
			t.Fatal(err)
		}
		return sent
	}
	before := digestOfLaptop()

	// The change merges; the policy is the default, record. `push anchor` sends the anchor...
	aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
	d.declared = nil
	if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
		t.Fatal(err)
	}
	// ...and its cascade leaves the laptop, saying so.
	var said bytes.Buffer
	d.declared = nil
	refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
	if err != nil || len(refused) != 0 {
		t.Fatalf("the cascade failed: %v %v", refused, err)
	}
	if len(d.declared) != 0 {
		t.Fatalf("the cascade sent %v a build its policy records", d.declared)
	}
	if digestOfLaptop() != before {
		t.Fatal("the laptop's last send moved: it was sent the held build")
	}
	for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
		"upgrade policy records", "`push laptop` sends it"} {
		if !strings.Contains(said.String(), want) {
			t.Errorf("the push did not say %q:\n%s", want, said.String())
		}
	}

	// Held and owed something else at once — a peer joined: still not sent, and both said: why it
	// is held, and that what else it is owed waits with it.
	aThirdMachine(t, open)
	d.declared = nil
	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
		compose, d, "", &said); err != nil {
		t.Fatal(err)
	}
	if len(d.declared) != 0 || digestOfLaptop() != before {
		t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
	}
	if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
		t.Fatalf("the push did not say both:\n%s", said.String())
	}

	// A policy that rolls out: the laptop is a consequence like any other, and sent.
	if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
		t.Fatal(err)
	}
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
		compose, d, "", &said); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
		t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
	}
	if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
		t.Fatalf("the new send did not record the new build: %v", builds)
	}
}

// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
// for another reason is sent by a push that names someone else.
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
	if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
		t.Fatal(err)
	}
	gens, err := generators(ctx, open)
	if err != nil {
		t.Fatal(err)
	}
	compose := composeForPush(open, gens)
	d := &recordedDelivery{inv: inv}
	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
		t.Fatal(err)
	}

	// `push spare`, the machine just placed: the others' peers change with it.
	aThirdMachine(t, open)
	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	var said bytes.Buffer
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
		t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
	}
	if strings.Contains(said.String(), "was not sent") {
		t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
	}

	// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
	record, err := inv.NodeByName(ctx, "laptop")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
		t.Fatal(err)
	}
	d.declared = nil
	said.Reset()
	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
		t.Fatal(err)
	}
	// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
	// question.
	if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
		t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
	}
}