package inventory import ( "errors" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" ) func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) { t.Helper() inv := fresh(t) ctx := t.Context() if _, err := inv.AddNode(ctx, "anchor"); err != nil { t.Fatal(err) } for _, m := range modules { if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { t.Fatal(err) } } return inv, "anchor" } // **Made once and kept.** A port that moved on every declaration would restart both ends each // time, and would hand a consumer a number that was true when it was read. func TestAPortIsAssignedOnceAndKept(t *testing.T) { inv, node := aNodeWithModules(t, "postgres") first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) if err != nil { t.Fatal(err) } second, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) if err != nil { t.Fatal(err) } if first.Machine != second.Machine { t.Fatalf("asking twice moved the port: %d then %d", first.Machine, second.Machine) } if first.Machine == 5432 { t.Error("the mesh handed back the port the module asked for, which is what it cannot know is free") } } // The fault this exists for: two modules wanting one number, which neither of them chose badly. func TestTwoModulesWantingOnePortGetTwo(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "another-database") a, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) if err != nil { t.Fatal(err) } b, err := inv.PortFor(t.Context(), node, "another-database", 5432, false) if err != nil { t.Fatal(err) } if a.Machine == b.Machine { t.Fatalf("both were put on %d, which is the collision this exists to prevent", a.Machine) } } // A port the protocol fixes is used as written, because a mail system elsewhere is not a mail // system. func TestAFixedPortIsTheOneTheProtocolSays(t *testing.T) { inv, node := aNodeWithModules(t, "mailu") got, err := inv.PortFor(t.Context(), node, "mailu", 25, true) if err != nil { t.Fatal(err) } if got.Machine != 25 { t.Fatalf("mail was put on %d", got.Machine) } if !got.Fixed { t.Error("it does not record that the protocol chose it, so nothing can refuse a second holder") } } // **A fixed port is a claim**: one holder per machine, refused by name at assignment rather than // by a container runtime at apply. func TestASecondModuleCannotHaveAFixedPort(t *testing.T) { inv, node := aNodeWithModules(t, "mailu", "other-mail") if _, err := inv.PortFor(t.Context(), node, "mailu", 25, true); err != nil { t.Fatal(err) } _, err := inv.PortFor(t.Context(), node, "other-mail", 25, true) if err == nil { t.Fatal("two modules were given port 25 on one machine") } if !errors.Is(err, ErrPortTaken) { t.Errorf("the refusal is not the one a caller can recognise: %v", err) } if !contains(err.Error(), "mailu") { t.Errorf("the refusal does not say who has it: %v", err) } } // An assigned port must not land on one the protocol fixed for something else. func TestAnAssignedPortAvoidsAFixedOne(t *testing.T) { inv, node := aNodeWithModules(t, "mailu", "web") fixed, err := inv.PortFor(t.Context(), node, "mailu", 20000, true) if err != nil { t.Fatal(err) } assigned, err := inv.PortFor(t.Context(), node, "web", 8080, false) if err != nil { t.Fatal(err) } if assigned.Machine == fixed.Machine { t.Fatalf("an assignment landed on %d, which the protocol had fixed for something else", fixed.Machine) } } // What a module gave back is available again. Otherwise a machine that ran a hundred modules over // a year has a hundred ports it cannot explain. func TestUnassigningGivesThePortBack(t *testing.T) { inv, node := aNodeWithModules(t, "postgres") first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) if err != nil { t.Fatal(err) } if err := inv.ReleasePorts(t.Context(), node, "postgres"); err != nil { t.Fatal(err) } held, err := inv.PortsFor(t.Context(), node) if err != nil { t.Fatal(err) } if len(held) != 0 { t.Fatalf("it still holds %v", held) } again, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) if err != nil { t.Fatal(err) } if again.Machine != first.Machine { t.Errorf("the freed port was not the first one offered again: %d then %d", first.Machine, again.Machine) } } func contains(s, what string) bool { return len(s) >= len(what) && (func() bool { for i := 0; i+len(what) <= len(s); i++ { if s[i:i+len(what)] == what { return true } } return false })() } // **The bug this whole thing is for** (novox/hq 04-ISSUES/028). The mesh keeps its own store on a // machine, from the bundle, before there is any mesh to ask. A database module assigned there was // handed 5432 — the port the store already had — and found out from a container runtime. func TestAModuleIsNotGivenAPortTheMachineAlreadyHolds(t *testing.T) { inv, node := aNodeWithModules(t, "some-service") ctx := t.Context() // What the machine says it raised for itself: the store, the broker, the control plane. if err := inv.RecordCarried(ctx, node, []int{5432, 5671, 8080, 20000, 20001}); err != nil { t.Fatal(err) } got, err := inv.PortFor(ctx, node, "some-service", 5432, false) if err != nil { t.Fatal(err) } for _, held := range []int{5432, 5671, 8080, 20000, 20001} { if got.Machine == held { t.Fatalf("it was given %d, which the machine already holds", held) } } if got.Machine != 20002 { t.Errorf("expected the lowest free one, 20002, and got %d", got.Machine) } } // A port the protocol fixes, already held by something the mesh did not put there, is refused — // and refused here rather than by a container runtime on the machine. func TestAFixedPortTheMachineAlreadyHoldsIsRefused(t *testing.T) { inv, node := aNodeWithModules(t, "mailu") ctx := t.Context() if err := inv.RecordCarried(ctx, node, []int{25}); err != nil { t.Fatal(err) } _, err := inv.PortFor(ctx, node, "mailu", 25, true) if err == nil { t.Fatal("a module was given a port something on the machine already holds") } if !contains(err.Error(), "already runs") { t.Errorf("the refusal does not say the machine itself has it: %v", err) } } // A machine that gave a port back is believed about that too. func TestWhatAMachineNoLongerHoldsIsAvailableAgain(t *testing.T) { inv, node := aNodeWithModules(t, "a-service") ctx := t.Context() if err := inv.RecordCarried(ctx, node, []int{20000}); err != nil { t.Fatal(err) } if err := inv.RecordCarried(ctx, node, nil); err != nil { t.Fatal(err) } got, err := inv.PortFor(ctx, node, "a-service", 1234, false) if err != nil { t.Fatal(err) } if got.Machine != 20000 { t.Errorf("a port the machine gave back was still reserved: got %d", got.Machine) } } // Unassigning a module gives its ports back — the fixed ones are what make this matter. // // Held past unassignment, port 25 stays claimed in the name of a mail system that is gone, and // every mail system after it is refused by a ghost. Found in review: ReleasePorts existed, was // documented "for when it is unassigned", and was called by nothing. func TestUnassigningReleasesTheModulesPorts(t *testing.T) { inv, node := aNodeWithModules(t, "mailu", "other-mail") ctx := t.Context() if _, err := inv.Assign(ctx, node, "mailu"); err != nil { t.Fatal(err) } if _, err := inv.PortFor(ctx, node, "mailu", 25, true); err != nil { t.Fatal(err) } if err := inv.Unassign(ctx, node, "mailu"); err != nil { t.Fatal(err) } if _, err := inv.Assign(ctx, node, "other-mail"); err != nil { t.Fatal(err) } if _, err := inv.PortFor(ctx, node, "other-mail", 25, true); err != nil { t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err) } } // novox/hq ADR 0100: a port a node was given for a module is the node's, and the mesh never hands // it to another. func TestAGivenPortIsNeverAssigned(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "web") ctx := t.Context() if err := inv.SetSettings(ctx, node, "postgres", map[string]any{catalogue.PortsSetting: map[string]any{"5432": 20000}}); err != nil { t.Fatal(err) } got, err := inv.PortFor(ctx, node, "web", 8080, false) if err != nil { t.Fatal(err) } if got.Machine == 20000 { t.Fatal("a port given to postgres was assigned to web") } if _, err := inv.PortFor(ctx, node, "web", 20000, true); !errors.Is(err, ErrPortTaken) { t.Fatalf("a fixed port given to another module was handed over: %v", err) } } // novox/hq ADR 0100: a given machine port has one holder. It is refused when another module has it, // assigned or given, when the same layer gives it twice, and when it is ssh's; and when it replaces // what the mesh assigned for that port, the assignment is given back. func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "web", "cache") ctx := t.Context() give := func(module string, ports map[string]any) error { return inv.SetSettings(ctx, node, module, map[string]any{catalogue.PortsSetting: ports}) } web, err := inv.PortFor(ctx, node, "web", 8080, false) if err != nil { t.Fatal(err) } if err := give("postgres", map[string]any{"5432": web.Machine}); !errors.Is(err, ErrPortTaken) { t.Fatalf("a port the mesh assigned to web was given to postgres: %v", err) } if err := give("postgres", map[string]any{"5432": 22}); err == nil { t.Fatal("ssh's port was given") } if err := give("postgres", map[string]any{"5432": 5433, "5433": 5433}); err == nil { t.Fatal("one machine port was given for two ports") } if err := give("cache", map[string]any{"6379": 6380}); err != nil { t.Fatal(err) } if err := give("postgres", map[string]any{"5432": 6380}); !errors.Is(err, ErrPortTaken) { t.Fatalf("a port given to cache was given to postgres: %v", err) } // Postgres was assigned a port for 5432; given one, the assignment is released and the number // is free again. assigned, err := inv.PortFor(ctx, node, "postgres", 5432, false) if err != nil { t.Fatal(err) } if err := give("postgres", map[string]any{"5432": 5433}); err != nil { t.Fatal(err) } // Given again, the same: its own given port is not a collision with itself. if err := give("postgres", map[string]any{"5432": 5433}); err != nil { t.Fatal(err) } held, err := inv.PortsFor(ctx, node) if err != nil { t.Fatal(err) } for _, a := range held { if a.Module == "postgres" { t.Fatalf("the assignment a given port replaced is still held: %+v", a) } } other, err := inv.PortFor(ctx, node, "cache", 11211, false) if err != nil { t.Fatal(err) } if other.Machine != assigned.Machine { t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine) } } // novox/hq ADR 0100: a port is a fact about one machine, so a layer for the whole mesh cannot give // one. Refused where it is set — stored, it refuses every node running the module at composition, // and the mesh cannot be pushed until somebody finds the layer that did it. func TestAPortGivenForTheWholeMeshIsRefusedWhereItIsSet(t *testing.T) { inv, node := aNodeWithModules(t, "postgres") ctx := t.Context() err := inv.SetSettings(ctx, "", "postgres", map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}) if err == nil || !strings.Contains(err.Error(), "per node") { t.Fatalf("a port given for the whole mesh was accepted: %v", err) } layers, err := inv.SettingsFor(ctx, node, "postgres") if err != nil { t.Fatal(err) } if len(layers) != 0 { t.Fatalf("the refused layer was stored: %v", layers) } // The same values for one machine are the ordinary setting. if err := inv.SetSettings(ctx, node, "postgres", map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { t.Fatal(err) } }