The mesh assigns the port, and a module says it once
novox/hq ADR 0038. A module cannot choose a port: it is written once and assigned anywhere, so any number it picks is a guess about a machine it has never seen. A database module met the mesh's own store on 5432 and was told, by a container runtime three layers down, that the port was already allocated. The number used to appear three times in every module — the rule set, what a consumer is told, and what the runtime publishes — agreeing only because one person wrote all three. Now it appears once, in `listens`, and the other two are derived: the container publishes `20000:5432`, the consumer is told 20000, and the rule set opens 20000. An assignment is made once and kept, as a credential is. A port that moved on every declaration would restart both ends each time and hand a consumer a number that was true when it was read. Ports the protocol fixes — mail on 25, submission on 587, DNS on 53 — say so, and are then claims: one holder per machine, and the second is refused by name at assignment. That is the mechanism the mesh already has for what is singular on a machine, pointed at ports. A mapping written the long way is left exactly as it is. Some things must be pinned by hand, and quietly overruling somebody who wrote both halves would be worse than not offering the short form. Still open, and known: the substrate is not a module, so the mesh has never heard of its own store and cannot yet assign around it. That is what 028 will still be about after this.
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
-- Which port a machine uses for what a module needs reachable.
|
||||
--
|
||||
-- novox/hq ADR 0038. A module cannot choose this: it is written once and assigned anywhere, so any
|
||||
-- number it picks is a guess about a machine it has never seen. Two modules guessing the same one
|
||||
-- is not a mistake either of them made -- it is a database module meeting the mesh's own store and
|
||||
-- being told, by a container runtime three layers down, that the port is already allocated.
|
||||
--
|
||||
-- **Made once and kept**, exactly as a credential is. A port that moved on every declaration would
|
||||
-- restart both ends each time, and would hand a consumer a number that was true when it was read.
|
||||
|
||||
create table port_assignment (
|
||||
node uuid not null references node(id) on delete cascade,
|
||||
module text not null references module(name) on delete cascade,
|
||||
|
||||
-- The port the software itself uses -- what a module writes down, and the only part it knows.
|
||||
wanted integer not null,
|
||||
-- What the machine publishes it on. The same as `wanted` when the protocol fixes it.
|
||||
machine integer not null,
|
||||
|
||||
-- Whether the protocol fixed it. Kept rather than derived: *this is 25 because it must be*
|
||||
-- and *this is 25 because it was free* are different facts, and only the first refuses a
|
||||
-- second holder.
|
||||
fixed boolean not null default false,
|
||||
|
||||
assigned_at timestamptz not null default now(),
|
||||
|
||||
primary key (node, module, wanted),
|
||||
|
||||
-- **One machine port, one holder.** The constraint is the point: a second module cannot be
|
||||
-- given a port the first has, and finding that out here is finding it out at assignment
|
||||
-- rather than at apply.
|
||||
unique (node, machine)
|
||||
);
|
||||
@@ -0,0 +1,186 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Which port a machine uses for what a module needs reachable.
|
||||
//
|
||||
// **A module cannot choose this** (novox/hq ADR 0038). It is written once and assigned anywhere,
|
||||
// so any number it picks is a guess about a machine it has never seen. The mesh is the only thing
|
||||
// that knows what else is there, so the mesh chooses — and a module says only that something must
|
||||
// be reachable, and what the software itself calls it.
|
||||
|
||||
// Assigned is where the machine puts one of a module's ports.
|
||||
type Assigned struct {
|
||||
Module string
|
||||
// Wanted is the port the software uses — what the module wrote down.
|
||||
Wanted int
|
||||
// Machine is where this machine publishes it.
|
||||
Machine int
|
||||
// Fixed means the protocol chose it, not the mesh.
|
||||
Fixed bool
|
||||
}
|
||||
|
||||
// The range the mesh assigns from.
|
||||
//
|
||||
// High and unprivileged, so an assignment never needs root and never lands on something a person
|
||||
// would recognise. Below the range Linux uses for outgoing connections, so an assignment cannot
|
||||
// collide with a port the kernel handed to something else while the machine was working.
|
||||
const (
|
||||
firstAssignable = 20000
|
||||
lastAssignable = 29999
|
||||
)
|
||||
|
||||
// ErrPortTaken is returned when a port the protocol fixes is already held by another module.
|
||||
var ErrPortTaken = errors.New("that port is already held on this machine")
|
||||
|
||||
// PortFor is where one module's port lives on one machine, choosing it the first time.
|
||||
//
|
||||
// **Kept once chosen.** A port that moved on every declaration would restart both ends each time,
|
||||
// and would hand a consumer a number that was true when it was read — which is the same argument
|
||||
// that makes a credential stable.
|
||||
func (i *Inventory) PortFor(
|
||||
ctx context.Context, node, module string, wanted int, fixed bool,
|
||||
) (Assigned, error) {
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return Assigned{}, err
|
||||
}
|
||||
|
||||
var held Assigned
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select machine, fixed from port_assignment
|
||||
where node = $1 and module = $2 and wanted = $3`,
|
||||
record.ID, module, wanted).Scan(&held.Machine, &held.Fixed)
|
||||
if err == nil {
|
||||
held.Module, held.Wanted = module, wanted
|
||||
// A module that has become fixed since it was assigned must move to the port the protocol
|
||||
// requires. Said rather than silently kept: a mail system on the port it was lent is a
|
||||
// mail system nothing can deliver to.
|
||||
if fixed && held.Machine != wanted {
|
||||
if err := i.freePort(ctx, record.ID, module, wanted); err != nil {
|
||||
return Assigned{}, err
|
||||
}
|
||||
return i.assignPort(ctx, record.ID, node, module, wanted, true)
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Assigned{}, err
|
||||
}
|
||||
return i.assignPort(ctx, record.ID, node, module, wanted, fixed)
|
||||
}
|
||||
|
||||
func (i *Inventory) freePort(ctx context.Context, node any, module string, wanted int) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`delete from port_assignment where node = $1 and module = $2 and wanted = $3`,
|
||||
node, module, wanted)
|
||||
return err
|
||||
}
|
||||
|
||||
func (i *Inventory) assignPort(
|
||||
ctx context.Context, nodeID any, node, module string, wanted int, fixed bool,
|
||||
) (Assigned, error) {
|
||||
taken, err := i.portsOn(ctx, nodeID)
|
||||
if err != nil {
|
||||
return Assigned{}, err
|
||||
}
|
||||
|
||||
machine := wanted
|
||||
if !fixed {
|
||||
// The lowest free one, so a machine's assignments are stable and readable rather than
|
||||
// scattered — and so the same set of modules on two machines gets the same numbers, which
|
||||
// makes a difference between two machines mean something.
|
||||
machine = 0
|
||||
for candidate := firstAssignable; candidate <= lastAssignable; candidate++ {
|
||||
if _, held := taken[candidate]; !held {
|
||||
machine = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if machine == 0 {
|
||||
return Assigned{}, fmt.Errorf(
|
||||
"%s has no free port left between %d and %d, which is ten thousand of them — "+
|
||||
"something is assigning ports it never gives back",
|
||||
node, firstAssignable, lastAssignable)
|
||||
}
|
||||
}
|
||||
|
||||
if by, held := taken[machine]; held {
|
||||
return Assigned{}, fmt.Errorf(
|
||||
"%w: %s needs %d and %s already has it on %s. A port the protocol fixes can have one "+
|
||||
"holder per machine, so one of them has to go somewhere else",
|
||||
ErrPortTaken, module, machine, by, node)
|
||||
}
|
||||
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into port_assignment (node, module, wanted, machine, fixed)
|
||||
values ($1, $2, $3, $4, $5)`,
|
||||
nodeID, module, wanted, machine, fixed)
|
||||
if err != nil {
|
||||
return Assigned{}, err
|
||||
}
|
||||
return Assigned{Module: module, Wanted: wanted, Machine: machine, Fixed: fixed}, nil
|
||||
}
|
||||
|
||||
/** Which machine ports are spoken for, and by whom. */
|
||||
func (i *Inventory) portsOn(ctx context.Context, nodeID any) (map[int]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select machine, module from port_assignment where node = $1`, nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[int]string{}
|
||||
for rows.Next() {
|
||||
var machine int
|
||||
var module string
|
||||
if err := rows.Scan(&machine, &module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[machine] = module
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PortsFor is every assignment a node holds, for composing its declaration.
|
||||
func (i *Inventory) PortsFor(ctx context.Context, node string) ([]Assigned, error) {
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select module, wanted, machine, fixed from port_assignment
|
||||
where node = $1 order by module, wanted`, record.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Assigned
|
||||
for rows.Next() {
|
||||
var a Assigned
|
||||
if err := rows.Scan(&a.Module, &a.Wanted, &a.Machine, &a.Fixed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ReleasePorts gives back everything a module held on a machine, for when it is unassigned.
|
||||
func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error {
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`delete from port_assignment where node = $1 and module = $2`, record.ID, module)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-control/internal/catalogue"
|
||||
)
|
||||
|
||||
func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) {
|
||||
t.Helper()
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range modules {
|
||||
if err := inv.RegisterModule(ctx,
|
||||
catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return inv, "anchor"
|
||||
}
|
||||
|
||||
// **Made once and kept.** A port that moved on every declaration would restart both ends each
|
||||
// time, and would hand a consumer a number that was true when it was read.
|
||||
func TestAPortIsAssignedOnceAndKept(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres")
|
||||
first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Machine != second.Machine {
|
||||
t.Fatalf("asking twice moved the port: %d then %d", first.Machine, second.Machine)
|
||||
}
|
||||
if first.Machine == 5432 {
|
||||
t.Error("the mesh handed back the port the module asked for, which is what it cannot know is free")
|
||||
}
|
||||
}
|
||||
|
||||
// The fault this exists for: two modules wanting one number, which neither of them chose badly.
|
||||
func TestTwoModulesWantingOnePortGetTwo(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "another-database")
|
||||
a, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := inv.PortFor(t.Context(), node, "another-database", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Machine == b.Machine {
|
||||
t.Fatalf("both were put on %d, which is the collision this exists to prevent", a.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
// A port the protocol fixes is used as written, because a mail system elsewhere is not a mail
|
||||
// system.
|
||||
func TestAFixedPortIsTheOneTheProtocolSays(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "mailu")
|
||||
got, err := inv.PortFor(t.Context(), node, "mailu", 25, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Machine != 25 {
|
||||
t.Fatalf("mail was put on %d", got.Machine)
|
||||
}
|
||||
if !got.Fixed {
|
||||
t.Error("it does not record that the protocol chose it, so nothing can refuse a second holder")
|
||||
}
|
||||
}
|
||||
|
||||
// **A fixed port is a claim**: one holder per machine, refused by name at assignment rather than
|
||||
// by a container runtime at apply.
|
||||
func TestASecondModuleCannotHaveAFixedPort(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "mailu", "other-mail")
|
||||
if _, err := inv.PortFor(t.Context(), node, "mailu", 25, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := inv.PortFor(t.Context(), node, "other-mail", 25, true)
|
||||
if err == nil {
|
||||
t.Fatal("two modules were given port 25 on one machine")
|
||||
}
|
||||
if !errors.Is(err, ErrPortTaken) {
|
||||
t.Errorf("the refusal is not the one a caller can recognise: %v", err)
|
||||
}
|
||||
if !contains(err.Error(), "mailu") {
|
||||
t.Errorf("the refusal does not say who has it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An assigned port must not land on one the protocol fixed for something else.
|
||||
func TestAnAssignedPortAvoidsAFixedOne(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "mailu", "web")
|
||||
fixed, err := inv.PortFor(t.Context(), node, "mailu", 20000, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assigned, err := inv.PortFor(t.Context(), node, "web", 8080, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if assigned.Machine == fixed.Machine {
|
||||
t.Fatalf("an assignment landed on %d, which the protocol had fixed for something else",
|
||||
fixed.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
// What a module gave back is available again. Otherwise a machine that ran a hundred modules over
|
||||
// a year has a hundred ports it cannot explain.
|
||||
func TestUnassigningGivesThePortBack(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres")
|
||||
first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.ReleasePorts(t.Context(), node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := inv.PortsFor(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(held) != 0 {
|
||||
t.Fatalf("it still holds %v", held)
|
||||
}
|
||||
again, err := inv.PortFor(t.Context(), node, "postgres", 5432, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Machine != first.Machine {
|
||||
t.Errorf("the freed port was not the first one offered again: %d then %d",
|
||||
first.Machine, again.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, what string) bool {
|
||||
return len(s) >= len(what) && (func() bool {
|
||||
for i := 0; i+len(what) <= len(s); i++ {
|
||||
if s[i:i+len(what)] == what {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})()
|
||||
}
|
||||
Reference in New Issue
Block a user