mesh-controller/internal/inventory/ports_test.go

package inventory

import (
	"errors"
	"testing"

	"github.com/novox/mesh-controller/internal/catalogue"
)

func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) {
	t.Helper()
	inv := fresh(t)
	ctx := t.Context()
	if _, err := inv.AddNode(ctx, "anchor"); err != nil {
		t.Fatal(err)
	}
	for _, m := range modules {
		if err := inv.RegisterModule(ctx,
			catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
			t.Fatal(err)
		}
	}
	return inv, "anchor"
}

// **Made once and kept.** A port that moved on every declaration would restart both ends each
// time, and would hand a consumer a number that was true when it was read.
func TestAPortIsAssignedOnceAndKept(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres")
	first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	if first.Machine != second.Machine {
		t.Fatalf("asking twice moved the port: %d then %d", first.Machine, second.Machine)
	}
	if first.Machine == 5432 {
		t.Error("the mesh handed back the port the module asked for, which is what it cannot know is free")
	}
}

// The fault this exists for: two modules wanting one number, which neither of them chose badly.
func TestTwoModulesWantingOnePortGetTwo(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres", "another-database")
	a, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	b, err := inv.PortFor(t.Context(), node, "another-database", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	if a.Machine == b.Machine {
		t.Fatalf("both were put on %d, which is the collision this exists to prevent", a.Machine)
	}
}

// A port the protocol fixes is used as written, because a mail system elsewhere is not a mail
// system.
func TestAFixedPortIsTheOneTheProtocolSays(t *testing.T) {
	inv, node := aNodeWithModules(t, "mailu")
	got, err := inv.PortFor(t.Context(), node, "mailu", 25, true)
	if err != nil {
		t.Fatal(err)
	}
	if got.Machine != 25 {
		t.Fatalf("mail was put on %d", got.Machine)
	}
	if !got.Fixed {
		t.Error("it does not record that the protocol chose it, so nothing can refuse a second holder")
	}
}

// **A fixed port is a claim**: one holder per machine, refused by name at assignment rather than
// by a container runtime at apply.
func TestASecondModuleCannotHaveAFixedPort(t *testing.T) {
	inv, node := aNodeWithModules(t, "mailu", "other-mail")
	if _, err := inv.PortFor(t.Context(), node, "mailu", 25, true); err != nil {
		t.Fatal(err)
	}
	_, err := inv.PortFor(t.Context(), node, "other-mail", 25, true)
	if err == nil {
		t.Fatal("two modules were given port 25 on one machine")
	}
	if !errors.Is(err, ErrPortTaken) {
		t.Errorf("the refusal is not the one a caller can recognise: %v", err)
	}
	if !contains(err.Error(), "mailu") {
		t.Errorf("the refusal does not say who has it: %v", err)
	}
}

// An assigned port must not land on one the protocol fixed for something else.
func TestAnAssignedPortAvoidsAFixedOne(t *testing.T) {
	inv, node := aNodeWithModules(t, "mailu", "web")
	fixed, err := inv.PortFor(t.Context(), node, "mailu", 20000, true)
	if err != nil {
		t.Fatal(err)
	}
	assigned, err := inv.PortFor(t.Context(), node, "web", 8080, false)
	if err != nil {
		t.Fatal(err)
	}
	if assigned.Machine == fixed.Machine {
		t.Fatalf("an assignment landed on %d, which the protocol had fixed for something else",
			fixed.Machine)
	}
}

// What a module gave back is available again. Otherwise a machine that ran a hundred modules over
// a year has a hundred ports it cannot explain.
func TestUnassigningGivesThePortBack(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres")
	first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.ReleasePorts(t.Context(), node, "postgres"); err != nil {
		t.Fatal(err)
	}
	held, err := inv.PortsFor(t.Context(), node)
	if err != nil {
		t.Fatal(err)
	}
	if len(held) != 0 {
		t.Fatalf("it still holds %v", held)
	}
	again, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	if again.Machine != first.Machine {
		t.Errorf("the freed port was not the first one offered again: %d then %d",
			first.Machine, again.Machine)
	}
}

func contains(s, what string) bool {
	return len(s) >= len(what) && (func() bool {
		for i := 0; i+len(what) <= len(s); i++ {
			if s[i:i+len(what)] == what {
				return true
			}
		}
		return false
	})()
}

// **The bug this whole thing is for** (novox/hq 04-ISSUES/028). The mesh keeps its own store on a
// machine, from the bundle, before there is any mesh to ask. A database module assigned there was
// handed 5432 — the port the store already had — and found out from a container runtime.
func TestAModuleIsNotGivenAPortTheMachineAlreadyHolds(t *testing.T) {
	inv, node := aNodeWithModules(t, "some-service")
	ctx := t.Context()

	// What the machine says it raised for itself: the store, the broker, the control plane.
	if err := inv.RecordCarried(ctx, node, []int{5432, 5671, 8080, 20000, 20001}); err != nil {
		t.Fatal(err)
	}

	got, err := inv.PortFor(ctx, node, "some-service", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	for _, held := range []int{5432, 5671, 8080, 20000, 20001} {
		if got.Machine == held {
			t.Fatalf("it was given %d, which the machine already holds", held)
		}
	}
	if got.Machine != 20002 {
		t.Errorf("expected the lowest free one, 20002, and got %d", got.Machine)
	}
}

// A port the protocol fixes, already held by something the mesh did not put there, is refused —
// and refused here rather than by a container runtime on the machine.
func TestAFixedPortTheMachineAlreadyHoldsIsRefused(t *testing.T) {
	inv, node := aNodeWithModules(t, "mailu")
	ctx := t.Context()
	if err := inv.RecordCarried(ctx, node, []int{25}); err != nil {
		t.Fatal(err)
	}
	_, err := inv.PortFor(ctx, node, "mailu", 25, true)
	if err == nil {
		t.Fatal("a module was given a port something on the machine already holds")
	}
	if !contains(err.Error(), "already runs") {
		t.Errorf("the refusal does not say the machine itself has it: %v", err)
	}
}

// A machine that gave a port back is believed about that too.
func TestWhatAMachineNoLongerHoldsIsAvailableAgain(t *testing.T) {
	inv, node := aNodeWithModules(t, "a-service")
	ctx := t.Context()
	if err := inv.RecordCarried(ctx, node, []int{20000}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordCarried(ctx, node, nil); err != nil {
		t.Fatal(err)
	}
	got, err := inv.PortFor(ctx, node, "a-service", 1234, false)
	if err != nil {
		t.Fatal(err)
	}
	if got.Machine != 20000 {
		t.Errorf("a port the machine gave back was still reserved: got %d", got.Machine)
	}
}

// Unassigning a module gives its ports back — the fixed ones are what make this matter.
//
// Held past unassignment, port 25 stays claimed in the name of a mail system that is gone, and
// every mail system after it is refused by a ghost. Found in review: ReleasePorts existed, was
// documented "for when it is unassigned", and was called by nothing.
func TestUnassigningReleasesTheModulesPorts(t *testing.T) {
	inv, node := aNodeWithModules(t, "mailu", "other-mail")
	ctx := t.Context()
	if err := inv.Assign(ctx, node, "mailu"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.PortFor(ctx, node, "mailu", 25, true); err != nil {
		t.Fatal(err)
	}
	if err := inv.Unassign(ctx, node, "mailu"); err != nil {
		t.Fatal(err)
	}
	if err := inv.Assign(ctx, node, "other-mail"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.PortFor(ctx, node, "other-mail", 25, true); err != nil {
		t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err)
	}
}

// novox/hq ADR 0100: a port a node was given for a module is the node's, and the mesh never hands
// it to another.
func TestAGivenPortIsNeverAssigned(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres", "web")
	ctx := t.Context()
	if err := inv.SetSettings(ctx, node, "postgres",
		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 20000}}); err != nil {
		t.Fatal(err)
	}
	got, err := inv.PortFor(ctx, node, "web", 8080, false)
	if err != nil {
		t.Fatal(err)
	}
	if got.Machine == 20000 {
		t.Fatal("a port given to postgres was assigned to web")
	}
	if _, err := inv.PortFor(ctx, node, "web", 20000, true); !errors.Is(err, ErrPortTaken) {
		t.Fatalf("a fixed port given to another module was handed over: %v", err)
	}
}

// novox/hq ADR 0100: a given machine port has one holder. It is refused when another module has it,
// assigned or given, when the same layer gives it twice, and when it is ssh's; and when it replaces
// what the mesh assigned for that port, the assignment is given back.
func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres", "web", "cache")
	ctx := t.Context()
	give := func(module string, ports map[string]any) error {
		return inv.SetSettings(ctx, node, module, map[string]any{catalogue.PortsSetting: ports})
	}

	web, err := inv.PortFor(ctx, node, "web", 8080, false)
	if err != nil {
		t.Fatal(err)
	}
	if err := give("postgres", map[string]any{"5432": web.Machine}); !errors.Is(err, ErrPortTaken) {
		t.Fatalf("a port the mesh assigned to web was given to postgres: %v", err)
	}
	if err := give("postgres", map[string]any{"5432": 22}); err == nil {
		t.Fatal("ssh's port was given")
	}
	if err := give("postgres", map[string]any{"5432": 5433, "5433": 5433}); err == nil {
		t.Fatal("one machine port was given for two ports")
	}
	if err := give("cache", map[string]any{"6379": 6380}); err != nil {
		t.Fatal(err)
	}
	if err := give("postgres", map[string]any{"5432": 6380}); !errors.Is(err, ErrPortTaken) {
		t.Fatalf("a port given to cache was given to postgres: %v", err)
	}

	// Postgres was assigned a port for 5432; given one, the assignment is released and the number
	// is free again.
	assigned, err := inv.PortFor(ctx, node, "postgres", 5432, false)
	if err != nil {
		t.Fatal(err)
	}
	if err := give("postgres", map[string]any{"5432": 5433}); err != nil {
		t.Fatal(err)
	}
	// Given again, the same: its own given port is not a collision with itself.
	if err := give("postgres", map[string]any{"5432": 5433}); err != nil {
		t.Fatal(err)
	}
	held, err := inv.PortsFor(ctx, node)
	if err != nil {
		t.Fatal(err)
	}
	for _, a := range held {
		if a.Module == "postgres" {
			t.Fatalf("the assignment a given port replaced is still held: %+v", a)
		}
	}
	other, err := inv.PortFor(ctx, node, "cache", 11211, false)
	if err != nil {
		t.Fatal(err)
	}
	if other.Machine != assigned.Machine {
		t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine)
	}
}