mesh-host/internal/bootstrap/ports.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/reachable"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/tunnel"
)

// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
//
// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's
// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests
// surfaces as a container that fails to bind, and one changed at genesis would be changed back when
// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the
// bundle, and handed to the controller as that node's setting for the module that binds it.
type FoundationPorts struct {
	Store      int `json:"store"`
	Bus        int `json:"bus"`
	AMQP       int `json:"amqp"`
	Management int `json:"management"`
	Registry   int `json:"registry"`
	Packages   int `json:"packages"`
	Hub        int `json:"hub"`
}

// DefaultPorts are the catalogue's numbers.
func DefaultPorts() FoundationPorts {
	return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000,
		Packages: 3000, Hub: 51820}
}

// DefaultOverlayRange is the controller's default private-network range.
const DefaultOverlayRange = "10.42.0.0/16"

// orDefaults fills every port left unsaid.
func (p FoundationPorts) orDefaults() FoundationPorts {
	d := DefaultPorts()
	for _, f := range []struct{ got, def *int }{
		{&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management},
		{&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub},
	} {
		if *f.got == 0 {
			*f.got = *f.def
		}
	}
	return p
}

// named is each port with what it is and its protocol, in a fixed order.
func (p FoundationPorts) named() []namedPort {
	return []namedPort{
		{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP},
		{"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry},
		{"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub},
	}
}

type namedPort struct {
	what, protocol string
	port           int
}

// Check refuses a port out of range, or one port given for two things.
func (p FoundationPorts) Check() error {
	seen := map[string]string{}
	for _, n := range p.named() {
		if n.port < 1 || n.port > 65535 {
			return fmt.Errorf("%s's port is %d, and a port is 1-65535", n.what, n.port)
		}
		key := n.protocol + "/" + strconv.Itoa(n.port)
		if other, twice := seen[key]; twice {
			return fmt.Errorf("%s and %s were both given %s", other, n.what, key)
		}
		seen[key] = n.what
	}
	return nil
}

// moduleSettings is what each foundation module is told about its ports on this node: the port it
// declares, to the machine's port it is given. Only what differs from the catalogue — a converged
// genesis on the defaults sets nothing, and so changes nothing it did before.
func (p FoundationPorts) moduleSettings() map[string]map[string]int {
	d := DefaultPorts()
	out := map[string]map[string]int{}
	add := func(module string, declared, given int) {
		if given == declared {
			return
		}
		if out[module] == nil {
			out[module] = map[string]int{}
		}
		out[module][strconv.Itoa(declared)] = given
	}
	add("postgres", d.Store, p.Store)
	add("lavinmq", d.Bus, p.Bus)
	add("lavinmq", d.AMQP, p.AMQP)
	add("lavinmq", d.Management, p.Management)
	add(RegistryModule, d.Registry, p.Registry)
	add(ForgeModule, d.Packages, p.Packages)
	return out
}

// PortsSetting is the controller's settings key for a module's given ports.
const PortsSetting = "ports"

// ServesSetting is the settings key that carries what a provider serves, inside the one binding a
// module carries rather than resolves — the builder's package binding (novox/hq 04-ISSUES/085).
//
// **A port given at genesis has to be a setting wherever it is read, or something puts it back.**
// The builder reaches the package registry through a binding written in its own manifest, because
// at genesis no module yet provides `package-registry` and so there is nothing for the controller
// to resolve it from. That binding used to be rewritten, as text, when the installer registered the
// builder — which a later registration from the catalogue undid silently. Set as a node's setting
// instead, it is held by the controller rather than by the manifest, so re-registering the builder
// leaves it where it was.
//
// **A settings layer is the module's, not one resource's.** The controller lays it over every file
// of that module which merges as JSON, so `serves` lands in the package binding only because that
// binding is the builder's one mergeable file. A second mergeable file added to the builder would
// be given a `serves` key too, meaning nothing to whatever reads it. Worth knowing before adding
// one.
const ServesSetting = "serves"

// setFoundationSettings tells the controller the ports this node gave a foundation module — and,
// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it
// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before
// the push that raises it, so the first declaration already names the node's ports.
func setFoundationSettings(ctx context.Context, o Options, control controlPlane, module string,
	say func(string)) error {
	values := map[string]any{}
	if ports := o.Ports.orDefaults().moduleSettings()[module]; len(ports) > 0 {
		values[PortsSetting] = ports
	}
	if o.Adopted && module == RegistryModule {
		values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"}
	}
	if packages := o.Ports.orDefaults().Packages; module == BuilderModule &&
		packages != DefaultPorts().Packages {
		// The one binding nothing resolves: the builder dials the forge by a number it carries.
		// The rest of the binding — the provision, the forge it is from, the account it presents —
		// is the manifest's and stays there.
		values[ServesSetting] = map[string]int{"port": packages}
	}
	if len(values) == 0 {
		return nil
	}
	raw, err := json.Marshal(values)
	if err != nil {
		return err
	}
	remote := "/" + module + "-settings.json"
	if err := control.carrying(ctx, module+"-settings.json", raw, remote); err != nil {
		return err
	}
	if _, err := control.tell(ctx, "settings", "set", module, remote, "--node", o.Node); err != nil {
		return err
	}
	say("  settings          " + module + " on " + o.Node + ": " + string(raw))
	return nil
}

// PortsRewrite says what RewritePorts changed.
type PortsRewrite struct {
	Places int
}

// RewritePorts puts the node's foundation ports into the produced bundle, in place of the
// template's, byte for byte like every other rewrite — so the file keeps its comments and a person
// can read what was applied. A port left at its default is not touched, so a genesis on the
// defaults produces exactly the bundle it did before.
//
// Only the machine's side moves: the outer port of each mapping, the addresses the control plane
// dials on the machine's loopback, and the address nodes are told to dial. What a container listens
// on inside itself, and what an action reaches inside the store's own network, stay as they are.
func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRewrite, error) {
	var out PortsRewrite
	p = p.orDefaults()
	d := DefaultPorts()
	bundle := r.Bundle
	var err error

	replace := func(from, to, what string) {
		if err != nil || from == to {
			return
		}
		bundle, err = replaceOnce(bundle, from, to, what)
		out.Places++
	}
	if p.Store != d.Store {
		replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port")
	}
	if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management {
		replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`,
			fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management),
			"the broker's published ports")
	}
	if err != nil {
		return out, err
	}

	// The control plane runs on the machine's network and dials the store and the broker on its
	// loopback, so its connection strings name the machine's ports. The schema step reaches the
	// store inside the store's own network and keeps the container's port — so these are found by
	// the control plane's environment, not by searching for the text.
	control, cerr := controlPlaneIn(r.Declaration)
	if cerr != nil {
		return out, cerr
	}
	for _, key := range sortedKeys(control.Env) {
		value := control.Env[key]
		now := value
		now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store))
		now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP))
		if strings.HasSuffix(now, "@127.0.0.1:15672") {
			now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management)
		}
		if key == brokerAddressVar {
			if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" {
				now = net.JoinHostPort(host, strconv.Itoa(p.Bus))
			}
		}
		if now != value {
			replace(`"`+key+`": "`+value+`"`, `"`+key+`": "`+now+`"`, "the control plane's "+key)
		}
	}
	if err != nil {
		return out, err
	}

	// The foundation's own filter, where the template carries one: it admits the bus and the
	// registry from anywhere, on whatever port they are.
	for _, f := range []struct{ def, now int }{{d.Bus, p.Bus}, {d.Registry, p.Registry}} {
		if f.def == f.now {
			continue
		}
		for _, form := range []string{"tcp dport %d accept", "ct original proto-dst %d accept"} {
			from, to := fmt.Sprintf(form, f.def), fmt.Sprintf(form, f.now)
			if n := bytes.Count(bundle, []byte(from)); n > 0 {
				bundle = bytes.ReplaceAll(bundle, []byte(from), []byte(to))
				out.Places += n
			}
		}
	}

	// The private network's range, when it is not the default, is the controller's to know.
	if overlayRange != "" && overlayRange != DefaultOverlayRange {
		replace(`"`+brokerAddressVar+`": `,
			`"MESH_OVERLAY_CIDR": "`+overlayRange+`",
        "`+brokerAddressVar+`": `, "where the control plane is told the private network's range")
		if err != nil {
			return out, err
		}
	}

	if out.Places == 0 {
		return out, nil
	}
	parsed, perr := declaration.ParseFileTrusted(bundle)
	if perr != nil {
		return out, fmt.Errorf("the bundle stopped being a declaration after its ports were rewritten, which is this installer's fault: %w", perr)
	}
	r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources)
	if c, cerr := controlPlaneIn(parsed); cerr == nil {
		r.BrokerAddress = c.Env[brokerAddressVar]
	}
	return out, nil
}

// PortsFree refuses a foundation port something else already holds, naming what holds it. What the
// mesh itself raised on an earlier run of genesis is not counted: ours says which holders are.
func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(reachable.Reach) bool) error {
	out, err := run(ctx, "ss", "-Hltunp")
	if err != nil {
		return fmt.Errorf("cannot read which ports this machine holds, so the foundation's cannot be checked free: %w", err)
	}
	sockets := reachable.Sockets(out)
	var published []reachable.Reach
	if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil {
		published = reachable.Published(ps)
	}
	held := reachable.Merge(sockets, published)

	var problems []string
	for _, n := range p.orDefaults().named() {
		var by []string
		for _, r := range held {
			if r.Protocol != n.protocol || r.Port != n.port || ours(r) {
				continue
			}
			holder := r.By
			if holder == "" {
				holder = "something ss does not name"
			}
			if r.Published {
				holder = "the container " + r.By
			}
			if !contains(by, holder) {
				by = append(by, holder)
			}
		}
		if len(by) > 0 {
			problems = append(problems, fmt.Sprintf("%s's port %s/%d is held by %s",
				n.what, n.protocol, n.port, strings.Join(by, ", ")))
		}
	}
	if len(problems) > 0 {
		return fmt.Errorf("the foundation's ports must be free before anything is raised:\n  - %s\n"+
			"Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+
			"--management-port, --registry-port, --packages-port, --hub-port); nothing was changed",
			strings.Join(problems, "\n  - "))
	}
	return nil
}

// TakeTheTunnel finds the tunnel an adopted machine's private network takes over (novox/hq ADR
// 0105) and settles the options on it: the hub's port is the tunnel's, the mesh's range is the
// tunnel's, and the interface is named for the enrolment that takes its key. Nil when no tunnel is
// up, which is an ordinary machine. A --hub-port or --overlay-range that disagrees with the
// tunnel is refused: the peers dial the tunnel's port and live in its range, and a mesh raised
// beside them on other numbers is the two-tunnel shape the record rejects.
func TakeTheTunnel(o *Options, run Runner) (*tunnel.Found, error) {
	found, err := tunnel.Find(context.Background(), tunnel.Runner(run), o.Tunnel)
	if errors.Is(err, tunnel.ErrNone) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("%w. An adopted hub takes over the tunnel it finds; nothing was changed", err)
	}
	if o.Ports.Hub != 0 && o.Ports.Hub != DefaultPorts().Hub && o.Ports.Hub != found.Port {
		return nil, fmt.Errorf("--hub-port %d disagrees with the tunnel %s, which listens on %d: the "+
			"private network takes over that tunnel on its own port, so leave --hub-port unsaid or "+
			"say %d", o.Ports.Hub, found.Interface, found.Port, found.Port)
	}
	if o.OverlayRange != "" && o.OverlayRange != DefaultOverlayRange && o.OverlayRange != found.Range {
		return nil, fmt.Errorf("--overlay-range %s disagrees with the tunnel %s, whose range is %s: the "+
			"private network takes over that tunnel with its range, so leave --overlay-range unsaid "+
			"or say %s", o.OverlayRange, found.Interface, found.Range, found.Range)
	}
	o.Tunnel = found.Interface
	o.Ports.Hub = found.Port
	o.OverlayRange = found.Range
	return &found, nil
}

// OverlayClear refuses a private-network range that overlaps an address or a route the machine
// already has — a predecessor's tunnel still running — naming the interface. The mesh's own
// interface is not counted.
func OverlayClear(ctx context.Context, run Runner, overlayRange string) error {
	if overlayRange == "" {
		overlayRange = DefaultOverlayRange
	}
	_, mine, err := net.ParseCIDR(overlayRange)
	if err != nil {
		return fmt.Errorf("the private network's range %q is not a range: %w", overlayRange, err)
	}
	var clashes []string
	if out, err := run(ctx, "ip", "-o", "addr", "show"); err == nil {
		for _, line := range strings.Split(out, "\n") {
			f := strings.Fields(line)
			// 3: wg0    inet 10.42.0.1/24 scope global wg0
			if len(f) < 4 || (f[2] != "inet" && f[2] != "inet6") {
				continue
			}
			iface := strings.TrimSuffix(f[1], ":")
			if clash(mine, f[3]) && iface != meshInterface {
				clashes = append(clashes, fmt.Sprintf("%s holds %s", iface, f[3]))
			}
		}
	} else {
		return fmt.Errorf("cannot read this machine's addresses to check the private network's range: %w", err)
	}
	if out, err := run(ctx, "ip", "-o", "route", "show"); err == nil {
		for _, line := range strings.Split(out, "\n") {
			f := strings.Fields(line)
			// 10.42.0.0/16 dev wg0 proto kernel scope link src 10.42.0.1
			if len(f) < 3 || f[0] == "default" {
				continue
			}
			iface := ""
			for i := range f {
				if f[i] == "dev" && i+1 < len(f) {
					iface = f[i+1]
				}
			}
			if iface != meshInterface && clash(mine, f[0]) {
				clashes = append(clashes, fmt.Sprintf("%s routes %s", iface, f[0]))
			}
		}
	}
	if len(clashes) > 0 {
		return fmt.Errorf("the private network's range %s overlaps what this machine already has: %s.\n"+
			"A tunnel a predecessor still runs would take the mesh's traffic. Give another range with "+
			"--overlay-range; nothing was changed", overlayRange, strings.Join(clashes, "; "))
	}
	return nil
}

// meshInterface is the private network's own interface, which a re-run finds holding its range.
const meshInterface = "mesh0"

func clash(mine *net.IPNet, other string) bool {
	if !strings.Contains(other, "/") {
		if ip := net.ParseIP(other); ip != nil {
			return mine.Contains(ip)
		}
		return false
	}
	ip, theirs, err := net.ParseCIDR(other)
	if err != nil {
		return false
	}
	return mine.Contains(theirs.IP) || theirs.Contains(mine.IP) || mine.Contains(ip)
}

// NamesFree refuses a foundation or bundle container name that a container already has, when no
// host made that container and this node has no record of it — a predecessor's container under the
// mesh's name, which raising the foundation would replace.
func NamesFree(ctx context.Context, run Runner, names []string, known store.State) error {
	var taken []string
	sorted := append([]string{}, names...)
	sort.Strings(sorted)
	for _, name := range sorted {
		// `container inspect`: a name is not unique across object kinds, and the bare form can
		// resolve to a same-named network or volume instead of reporting the container absent.
		out, err := run(ctx, "docker", "container", "inspect", "--format",
			"{{index .Config.Labels \"mesh-host.spec\"}}", name)
		if err != nil {
			continue // no such container
		}
		label := strings.TrimSpace(out)
		if label != "" && label != "<no value>" {
			continue
		}
		if known.Recorded(string(declaration.TypeContainer), name) {
			continue
		}
		if name == giteaBootstrap && len(known.Resources) > 0 {
			// Genesis raises the package registry itself, by hand and before the host records
			// anything of it, so on a re-run it is found under its own name with no label and no
			// record. A machine that carries what an earlier genesis raised made it.
			continue
		}
		taken = append(taken, name)
	}
	if len(taken) > 0 {
		return fmt.Errorf("this machine already runs a container under the name the foundation uses, "+
			"and nothing of the mesh's made it: %s.\nRaising the foundation would replace it. Rename or "+
			"stop it first; nothing was changed", strings.Join(taken, ", "))
	}
	return nil
}

// CheckTheMachine is every check genesis makes before raising anything that this machine does not
// already hold what the foundation needs: its ports, its private network's range, its containers'
// names (novox/hq ADR 0100). A re-run of genesis finds the foundation it raised and does not count
// it.
func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declaration.Declaration,
	say func(string)) error {
	known, err := store.Load(o.State)
	if err != nil {
		return err
	}
	rerun := len(known.Resources) > 0
	names := foundationNames(bundle)
	mine := map[string]bool{}
	for _, n := range names {
		mine[n] = true
	}
	p := o.Ports.orDefaults()
	ours := func(r reachable.Reach) bool {
		switch {
		case mine[r.By]:
			return true
		case !rerun:
			return false
		case r.By == "gitea" && r.Port == p.Packages:
			// The package registry runs on the machine's network, so ss names its process.
			return true
		case r.By == "" && r.Protocol == "udp" && r.Port == p.Hub:
			// The private network's hub is a kernel interface and has no process.
			return true
		}
		return false
	}
	if o.Tunnel != "" {
		// The hub's port is the found tunnel's, held by that tunnel until the mesh's interface
		// takes it over (novox/hq ADR 0105): held by design, not by something else.
		inner := ours
		ours = func(r reachable.Reach) bool {
			return inner(r) || (r.Protocol == "udp" && r.Port == p.Hub)
		}
	}
	if err := PortsFree(ctx, run, p, ours); err != nil {
		return err
	}
	say(fmt.Sprintf("  ports free        store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
		p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
	if o.Tunnel != "" {
		// One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the
		// two must not overlap applies only where a found tunnel is left running beside the mesh's
		// (ADR 0100, narrowed by ADR 0105).
		say(fmt.Sprintf("  range             %s is the tunnel %s's, taken over; not checked against it",
			o.OverlayRange, o.Tunnel))
	} else if err := OverlayClear(ctx, run, o.OverlayRange); err != nil {
		return err
	}
	if err := NamesFree(ctx, run, names, known); err != nil {
		return err
	}
	return nil
}

// foundationNames are the containers the foundation and genesis raise under fixed names.
func foundationNames(bundle *declaration.Declaration) []string {
	names := []string{ControlPlaneModule, giteaBootstrap, "mesh-registry"}
	for _, n := range containerNames(bundle) {
		if !contains(names, n) {
			names = append(names, n)
		}
	}
	sort.Strings(names)
	return names
}