mesh-controller/cmd/mesh-controller/network.go

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/overlay"
)

// the private network: who is on it, where, and what they are called.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.

func overlayCIDR() string {
	if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
		return v
	}
	return "10.42.0.0/16"
}

func overlayCommand(ctx context.Context, args []string) error {
	if len(args) == 0 {
		return errors.New("overlay place <node> [flags], or overlay show")
	}
	// Answered before anything is opened. A message about which command to use should not need a
	// database to say so, and needing one turns a redirect into a connection error.
	if args[0] == "push" {
		return errors.New("`overlay push` is now `push`, which sends a node its network AND " +
			"what its assignments resolve to — the two are computed from one picture of the " +
			"mesh, and sending them separately would let them disagree")
	}
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	switch args[0] {
	case "place":
		return overlayPlace(ctx, inv, args[1:])
	case "show":
		return overlayShow(ctx, open)

	default:
		return fmt.Errorf("overlay has no %q; it has place and show", args[0])
	}
}

func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
	if len(args) == 0 {
		return errors.New(
			"overlay place <node> [--endpoint host:port] [--site name] [--hub], or --nothing")
	}
	node := args[0]

	set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
	endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
	site := set.String("site", "", "where this machine physically is, or empty if it roams")
	hub := set.Bool("hub", false, "this node is the hub every other routes through")
	nothing := set.Bool("nothing", false,
		"place it with nothing set: not dialable, no site, not the hub")
	if err := set.Parse(args[1:]); err != nil {
		return err
	}

	// **All three are declared together, so saying nothing took all three away.** The sibling of
	// `node public-domain`: `overlay place anchor` reads like it places the node it names, and it
	// silently unset the endpoint every other machine dials, the site it is in, and the hub if it
	// was the hub — every path through it going with them, at the moment somebody was trying to
	// look at it.
	//
	// A placement with nothing set is a real thing to want — a machine that roams and opens every
	// path itself is exactly that — so it keeps a way to say so, by name.
	if set.NFlag() == 0 {
		return fmt.Errorf("overlay place %s was given nothing to place it with, and all three are "+
			"declared together — it would take away the endpoint other machines dial %s at, its "+
			"site, and the hub if it is the hub. Say --endpoint/--site/--hub, or --nothing if that "+
			"is what you meant", node, node)
	}
	if *nothing && (*endpoint != "" || *site != "" || *hub) {
		return fmt.Errorf("give %s a placement or --nothing, not both: they say opposite things "+
			"and the mesh will not choose between them", node)
	}

	// One hub per mesh, refused rather than last-write-wins: with two flagged, which one the
	// graph and the broker address pick is order-dependent — the silent-election fault ADR 0007
	// exists to avoid, one flag over (novox/hq issue 059). Moving the hub is explicit: re-place
	// the old one without --hub first.
	if *hub {
		placed, err := inv.Overlays(ctx)
		if err != nil {
			return err
		}
		for _, p := range placed {
			if p.Hub && p.Name != node {
				return fmt.Errorf("%s is already the hub; a mesh has one. Re-place %s without "+
					"--hub first if the hub is moving", p.Name, p.Name)
			}
		}
	}

	// Declared, all three. The address is evidence of reachability and is not the fact, and hub
	// election by address prefix fails silently (novox/hq ADR 0007).
	if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
		return err
	}
	found, err := inv.NodeByName(ctx, node)
	if err != nil {
		return err
	}
	address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
	if err != nil {
		return err
	}

	fmt.Printf("%s is at %s on the overlay\n", node, address)
	fmt.Println("  credentials issued for it before this placement keep their old broker address —" +
		" `module issue` them again and push (novox/hq issue 059)")
	switch {
	case *hub:
		fmt.Println("  the hub — every node not sharing a site routes through it")
	case *endpoint == "":
		fmt.Println("  not dialable — it opens every path itself")
	}
	if *site != "" {
		fmt.Printf("  at %s, so it peers directly with anything else there\n", *site)
	}
	return nil
}

// network builds the private network over the machines that resolved the module for it.
//
// Not over every node the mesh knows. **A machine is on the private network because it was given
// the module**, and one that was not is absent from every peer list and from the names — which is
// the only thing "not on the network" can mean. Until this, having an address was enough, and
// there was no way to keep a machine off.
//
// Every node at once, which is the whole reason this is the control plane's work: a peer list is
// derived from all the others, so no node could compute its own.
func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
	refused map[string]string) (*overlay.Generator, error) {
	places, err := inv.Overlays(ctx)
	if err != nil {
		return nil, err
	}
	nodes := make([]overlay.Node, 0, len(places))
	for _, p := range places {
		if !on[p.Name] {
			continue
		}
		nodes = append(nodes, overlay.Node{
			Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
			Site: p.Site, Hub: p.Hub, Address: p.Address,
		})
	}
	if len(nodes) == 0 {
		// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
		// answers rather than refusing -- Compute would refuse for want of a hub, and reporting
		// "no hub" to somebody who never asked for a network would be a lie about the cause.
		return overlay.Empty(), nil
	}
	g, err := overlay.From(nodes, overlayCIDR(), "")
	if g != nil {
		// The artifact store, as this network reaches it. Found rather than configured: the
		// provider is whichever module offers it, on whichever machine holds that module — and if
		// nothing does yet (genesis raises the registry before the catalogue knows it), there is
		// no trust to write and nothing is written (novox/hq ADR 0082).
		//
		// Refused rather than composed without it when the question could not be answered: a
		// declaration missing the trust because a lookup failed is a machine that cannot pull,
		// delivered by a push that reported success — and nothing recomposes it until the next
		// push (the shape of novox/hq issues 042/048, reappearing as a race).
		at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
		if storeErr != nil {
			return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
		}
		if found {
			g.TrustRegistry(overlay.InternalName(at) + ":" + port)
		}
	}
	if err != nil && len(refused) > 0 {
		// The network is missing something, and some machines could not be resolved at all. Those
		// are almost always the same fact: a node that does not resolve contributes nothing, so
		// reporting "no hub" would name a consequence and hide the cause.
		var who []string
		for name, why := range refused {
			who = append(who, fmt.Sprintf("  %s: %s", name, why))
		}
		sort.Strings(who)
		return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+
			"probably why:\n%s", err, len(refused), strings.Join(who, "\n"))
	}
	return g, err
}

// graph is the whole mesh's network, for showing it.
func graph(ctx context.Context, open *stores) ([]overlay.Node, overlay.Graph, error) {
	inv := open.inventory
	on, refused, err := whoResolves(ctx, open, overlay.Requirement)
	if err != nil {
		return nil, nil, err
	}
	g, err := network(ctx, inv, on, refused)
	if err != nil {
		return nil, nil, err
	}
	return g.Nodes(), g.Graph(), nil
}

// whoResolves is the machines whose resolution answers a requirement, and why the others did not.
//
// By what a module **provides**, not by its name. WireGuard is one way to have a private network
// and there could be others, so a machine is on the network because something it runs provides
// one — asking for a particular module by name would be the mistake this whole mechanism exists
// to avoid.
//
// Resolved rather than read from the assignment table, because a module can arrive by being
// required by something else, and a machine that needs the private network to do its job is on it
// for the same reason as one that was handed it directly.
func whoResolves(ctx context.Context, open *stores, requirement string) (
	map[string]bool, map[string]string, error) {
	inv := open.inventory
	nodes, err := inv.Nodes(ctx)
	if err != nil {
		return nil, nil, err
	}
	on := map[string]bool{}
	// Why a node could not be resolved, kept rather than raised: one broken node must not stop
	// the rest being described, and whoever is rendering that node will raise it themselves.
	refused := map[string]string{}
	for _, n := range nodes {
		plan, _, err := planFor(ctx, open, n.Name)
		if err != nil {
			refused[n.Name] = err.Error()
			continue
		}
		for _, m := range plan.Modules {
			for _, offered := range m.Offers() {
				if offered == requirement {
					on[n.Name] = true
				}
			}
		}
	}
	return on, refused, nil
}

// rendering is everything a declaration needs, computed over the whole mesh.
func generators(ctx context.Context, open *stores) (
	map[string]catalogue.Generator, error) {
	inv := open.inventory
	on, refused, err := whoResolves(ctx, open, overlay.Addressing)
	if err != nil {
		return nil, err
	}
	net, err := network(ctx, inv, on, refused)
	if err != nil {
		return nil, err
	}
	// **One generator now.** Two more used to sit beside it — the names and a resolver's zone
	// file — as modules that ran nothing. Both are facts a module asks for in its manifest
	// (`facts:` — catalogue.FactsInto), computed from the same machines this sees: the ones on the
	// private network, because a name for a machine not on it would resolve to an address nothing
	// can reach.
	return map[string]catalogue.Generator{
		overlay.Name: net,
	}, nil
}

func overlayShow(ctx context.Context, open *stores) error {
	nodes, computed, err := graph(ctx, open)
	if err != nil {
		return err
	}
	if len(nodes) == 0 {
		// Not "this mesh has no nodes", which it said until the network became a module and was
		// then a lie about the cause: a mesh can have every node it will ever have and nobody on
		// the private network, because nobody asked for one.
		fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n",
			overlay.Name)
		return nil
	}

	for _, n := range nodes {
		place := n.Address
		if place == "" {
			// Said, not skipped. A node with no place is a node with no network, and it should
			// be visible here rather than quietly absent from a list of who is on it.
			place = "no address — run `overlay place`"
		}
		fmt.Printf("%-16s %-14s", n.Name, place)
		switch {
		case n.Hub:
			fmt.Print("  hub")
		case !n.Reachable():
			fmt.Print("  not dialable")
		}
		if n.Site != "" {
			fmt.Printf("  at %s", n.Site)
		}
		fmt.Println()
		for _, p := range computed[n.Name] {
			fmt.Printf("    → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
		}
	}
	return nil
}

// SilentFor is how long a node may be quiet before the mesh says so.
//
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
// is not the point — being able to say "out of touch" at all is, and nothing could before.
const SilentFor = 3 * time.Minute

// whereEveryoneIs is each machine's name on the private network, for the ones on it.
//
// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every
// other path here answers a question about one node by resolving the others; this one is called
// *from* that path, so doing the same would not terminate — which it did not, for two minutes,
// until it was run.
//
// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the
// private network depends on what it was assigned and what that requires, both of which are local
// facts. What it takes *from* other machines does not change the answer.
//
// The distinction that matters is kept: a machine absent from the network module's own view is
// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the
// network became something a machine is given.
func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
	shelf map[string]catalogue.Manifest) (map[string]string, error) {

	if shelf == nil {
		// Refused rather than answered. Being on the private network is a conclusion about what a
		// node resolves to, so with no catalogue nothing resolves and the honest answer is
		// "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused
		// every certificate the mesh was asked for while saying the machine was on no network.
		return nil, errors.New(
			"asked where everyone is without the catalogue, which cannot be answered")
	}
	places, err := onTheNetwork(ctx, inv, shelf)
	if err != nil {
		return nil, err
	}
	out := map[string]string{}
	for _, p := range places {
		out[p.Name] = overlay.InternalName(p.Name)
	}
	return out, nil
}

// onTheNetwork is every placed machine that resolves the private network — has an address AND
// runs what puts it there. "Has an address" alone was true of every placed machine and told you
// nothing about whether anything could reach it; a name written for such a machine resolves to
// an address that does not answer, and a connection to it hangs (novox/hq issue 079).
func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
	shelf map[string]catalogue.Manifest) ([]inventory.Overlay, error) {
	places, err := inv.Overlays(ctx)
	if err != nil {
		return nil, err
	}
	var out []inventory.Overlay
	for _, p := range places {
		if p.Address == "" {
			continue
		}
		assigned, err := inv.Assigned(ctx, p.Name)
		if err != nil || len(assigned) == 0 {
			continue
		}
		caps, _ := inv.ProfileOf(ctx, p.Name)
		got, err := catalogue.Resolve(shelf, assigned,
			catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
			catalogue.World{Unchecked: true})
		if err != nil {
			continue
		}
		for _, m := range got.Modules {
			for _, offered := range m.Offers() {
				if offered == overlay.Requirement {
					out = append(out, p)
				}
			}
		}
	}
	return out, nil
}

// onThePrivateNetwork is every node's address on the private network, sorted — the same set
// the names and the resolver mean by it (onTheNetwork), so a rule saying "from the mesh" admits
// exactly the machines the mesh names.
//
// A node with no address is left out rather than rendered as an empty source: an empty entry in a
// source set is a syntax error in the rule file, and a rule file that does not load leaves the
// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule,
// because nothing reports it.
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory,
	shelf map[string]catalogue.Manifest) ([]string, error) {
	places, err := onTheNetwork(ctx, inv, shelf)
	if err != nil {
		return nil, err
	}
	var out []string
	for _, p := range places {
		if strings.TrimSpace(p.Address) != "" {
			out = append(out, p.Address)
		}
	}
	sort.Strings(out)
	return out, nil
}

// namesInTheMesh is every machine's internal name and the address behind it — every machine
// that is on the private network, the same set the resolver means by that.
//
// A machine that is not has no name: writing one that resolves to nothing is worse than not
// writing it, because a connection to an address that does not answer hangs where a name that
// does not resolve fails at once and says so. A machine placed on the overlay but not running
// the module that puts it there is exactly that (novox/hq issue 079).
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
	shelf map[string]catalogue.Manifest) (map[string]string, error) {
	places, err := onTheNetwork(ctx, inv, shelf)
	if err != nil {
		return nil, err
	}
	out := map[string]string{}
	for _, p := range places {
		out[overlay.InternalName(p.Name)] = p.Address
	}
	return out, nil
}

// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
// module providing it is assigned to a machine that is on the private network.
//
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
// and nothing recomposed the machine until the next push. "No store" must mean the mesh has none,
// not that the question went unanswered.
func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
	on map[string]bool) (node, port string, found bool, err error) {

	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
	}
	providers := map[string]string{} // module -> served port
	for name, m := range shelf {
		served, offers := m.Serves[catalogue.ArtifactStoreProvision]
		if !offers {
			continue
		}
		if p, ok := served["port"]; ok {
			providers[name] = fmt.Sprintf("%v", p)
		}
	}
	if len(providers) == 0 {
		return "", "", false, nil
	}
	for machine := range on {
		assigned, err := inv.Assigned(ctx, machine)
		if err != nil {
			return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
		}
		for _, a := range assigned {
			if p, ok := providers[a]; ok {
				return machine, p, true, nil
			}
		}
	}
	return "", "", false, nil
}