mesh-controller/internal/inventory/ports.go

package inventory

import (
	"context"
	"errors"
	"fmt"

	"github.com/jackc/pgx/v5"
)

// Which port a machine uses for what a module needs reachable.
//
// **A module cannot choose this** (novox/hq ADR 0038). It is written once and assigned anywhere,
// so any number it picks is a guess about a machine it has never seen. The mesh is the only thing
// that knows what else is there, so the mesh chooses — and a module says only that something must
// be reachable, and what the software itself calls it.

// Assigned is where the machine puts one of a module's ports.
type Assigned struct {
	Module string
	// Wanted is the port the software uses — what the module wrote down.
	Wanted int
	// Machine is where this machine publishes it.
	Machine int
	// Fixed means the protocol chose it, not the mesh.
	Fixed bool
}

// The range the mesh assigns from.
//
// High and unprivileged, so an assignment never needs root and never lands on something a person
// would recognise. Below the range Linux uses for outgoing connections, so an assignment cannot
// collide with a port the kernel handed to something else while the machine was working.
const (
	firstAssignable = 20000
	lastAssignable  = 29999
)

// ErrPortTaken is returned when a port the protocol fixes is already held by another module.
var ErrPortTaken = errors.New("that port is already held on this machine")

// PortFor is where one module's port lives on one machine, choosing it the first time.
//
// **Kept once chosen.** A port that moved on every declaration would restart both ends each time,
// and would hand a consumer a number that was true when it was read — which is the same argument
// that makes a credential stable.
func (i *Inventory) PortFor(
	ctx context.Context, node, module string, wanted int, fixed bool,
) (Assigned, error) {
	record, err := i.NodeByName(ctx, node)
	if err != nil {
		return Assigned{}, err
	}

	var held Assigned
	err = i.store.Pool().QueryRow(ctx,
		`select machine, fixed from port_assignment
		 where node = $1 and module = $2 and wanted = $3`,
		record.ID, module, wanted).Scan(&held.Machine, &held.Fixed)
	if err == nil {
		held.Module, held.Wanted = module, wanted
		// A module that has become fixed since it was assigned must move to the port the protocol
		// requires. Said rather than silently kept: a mail system on the port it was lent is a
		// mail system nothing can deliver to.
		if fixed && held.Machine != wanted {
			if err := i.freePort(ctx, record.ID, module, wanted); err != nil {
				return Assigned{}, err
			}
			return i.assignPort(ctx, record.ID, node, module, wanted, true)
		}
		return held, nil
	}
	if !errors.Is(err, pgx.ErrNoRows) {
		return Assigned{}, err
	}
	return i.assignPort(ctx, record.ID, node, module, wanted, fixed)
}

func (i *Inventory) freePort(ctx context.Context, node any, module string, wanted int) error {
	_, err := i.store.Pool().Exec(ctx,
		`delete from port_assignment where node = $1 and module = $2 and wanted = $3`,
		node, module, wanted)
	return err
}

func (i *Inventory) assignPort(
	ctx context.Context, nodeID any, node, module string, wanted int, fixed bool,
) (Assigned, error) {
	taken, err := i.portsOn(ctx, nodeID)
	if err != nil {
		return Assigned{}, err
	}

	machine := wanted
	if !fixed {
		// The lowest free one, so a machine's assignments are stable and readable rather than
		// scattered — and so the same set of modules on two machines gets the same numbers, which
		// makes a difference between two machines mean something.
		machine = 0
		for candidate := firstAssignable; candidate <= lastAssignable; candidate++ {
			if _, held := taken[candidate]; !held {
				machine = candidate
				break
			}
		}
		if machine == 0 {
			return Assigned{}, fmt.Errorf(
				"%s has no free port left between %d and %d, which is ten thousand of them — "+
					"something is assigning ports it never gives back",
				node, firstAssignable, lastAssignable)
		}
	}

	if by, held := taken[machine]; held {
		return Assigned{}, fmt.Errorf(
			"%w: %s needs %d and %s already has it on %s. A port the protocol fixes can have one "+
				"holder per machine, so one of them has to go somewhere else",
			ErrPortTaken, module, machine, by, node)
	}

	_, err = i.store.Pool().Exec(ctx,
		`insert into port_assignment (node, module, wanted, machine, fixed)
		 values ($1, $2, $3, $4, $5)`,
		nodeID, module, wanted, machine, fixed)
	if err != nil {
		return Assigned{}, err
	}
	return Assigned{Module: module, Wanted: wanted, Machine: machine, Fixed: fixed}, nil
}

/** Which machine ports are spoken for, and by whom. */
func (i *Inventory) portsOn(ctx context.Context, nodeID any) (map[int]string, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select machine, module from port_assignment where node = $1`, nodeID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	out := map[int]string{}
	for rows.Next() {
		var machine int
		var module string
		if err := rows.Scan(&machine, &module); err != nil {
			return nil, err
		}
		out[machine] = module
	}
	return out, rows.Err()
}

// PortsFor is every assignment a node holds, for composing its declaration.
func (i *Inventory) PortsFor(ctx context.Context, node string) ([]Assigned, error) {
	record, err := i.NodeByName(ctx, node)
	if err != nil {
		return nil, err
	}
	rows, err := i.store.Pool().Query(ctx,
		`select module, wanted, machine, fixed from port_assignment
		 where node = $1 order by module, wanted`, record.ID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var out []Assigned
	for rows.Next() {
		var a Assigned
		if err := rows.Scan(&a.Module, &a.Wanted, &a.Machine, &a.Fixed); err != nil {
			return nil, err
		}
		out = append(out, a)
	}
	return out, rows.Err()
}

// ReleasePorts gives back everything a module held on a machine, for when it is unassigned.
func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error {
	record, err := i.NodeByName(ctx, node)
	if err != nil {
		return err
	}
	_, err = i.store.Pool().Exec(ctx,
		`delete from port_assignment where node = $1 and module = $2`, record.ID, module)
	return err
}