mesh-controller/cmd/mesh-control/plan.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/novox/mesh-control/internal/catalogue"
	"github.com/novox/mesh-control/internal/inventory"
	"github.com/novox/mesh-control/internal/licences"
)

// working out what one machine should be.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.

// planFor works out everything a node should run, from what was assigned to it.
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
	inv := open.inventory
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}
	assigned, err := inv.Assigned(ctx, nodeName)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}
	capabilities, err := inv.ProfileOf(ctx, nodeName)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}

	places, err := inv.Overlays(ctx)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}
	var site string
	for _, p := range places {
		if p.Name == nodeName {
			site = p.Site
		}
	}

	world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}
	world.Pinned, err = inv.PinsFor(ctx, nodeName)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}

	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}

	// What this mesh can answer with a record rather than a machine, and which record each of
	// this node's modules was put on. Read across a context boundary by name, which is what
	// crossing one is allowed to carry (novox/hq ADR 0008).
	world.Licences, world.Using, err = licencesFor(ctx, open, nodeName)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}

	resolved, err := catalogue.Resolve(shelf, assigned,
		catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
			At: onNetwork[nodeName]}, world)
	if err != nil {
		return catalogue.Resolution{}, nil, err
	}

	// The credential for each thing this node takes from elsewhere. Made once and kept, so the
	// password a provider is told to create is the one its consumer was given — and sealed to
	// this node before it was ever written down, so nothing between here and there can read it.
	for i, n := range resolved.Needs {
		if n.ByRecord {
			// Answered by something the mesh holds, so there is no pair-wise secret between two
			// machines. Its key was supplied by a person and sealed to this node then; the mesh
			// discarded the plaintext and cannot make another.
			sealed, err := keyFor(ctx, open, n.From, nodeName, n.For)
			if err != nil {
				return catalogue.Resolution{}, nil, err
			}
			resolved.Needs[i].Sealed = sealed
			continue
		}
		secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From)
		if err != nil {
			// Said rather than skipped. A machine that resolves cleanly and receives no
			// credential is one that will fail to authenticate at some later, less obvious
			// moment.
			return catalogue.Resolution{}, nil, fmt.Errorf(
				"%s on %s needs %s from %s and no credential could be made for it: %w",
				n.For, nodeName, n.Name, n.From, err)
		}
		resolved.Needs[i].Sealed = secret.ForConsumer
	}

	// Settings for everything that resolved, including modules nobody assigned directly: a
	// requirement pulled in by something else is still configurable, and finding out that it is
	// not only when you try would be an arbitrary line nobody could predict.
	settings := catalogue.SettingsBy{}
	var stray []string
	for _, m := range resolved.Modules {
		layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
		if err != nil {
			return catalogue.Resolution{}, nil, err
		}
		if len(layers) == 0 {
			continue
		}
		settings[m.Module] = layers
		stray = append(stray, catalogue.UnusedSettings(m, layers)...)
	}
	if len(stray) > 0 {
		// Somebody set something that reaches no file. Said here rather than discovered by the
		// machine not behaving differently, which is the slowest way there is.
		return catalogue.Resolution{}, nil, fmt.Errorf(
			"these settings reach nothing:\n  - %s", strings.Join(stray, "\n  - "))
	}
	return resolved, settings, nil
}

// theRestOfTheMesh is what every other node holds and offers.
//
// Two things at once because they come from the same place — resolving the other nodes — and
// because both are facts about what is actually running rather than records that could disagree
// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node
// runs; neither is a table somebody keeps up to date.
//
// **Two passes over the others.** What a node offers the mesh needs that node resolved, and
// resolving it may need what the mesh offers. So the first pass takes brokered requirements on
// trust and answers only *what does each node offer*; the second answers everything with that in
// hand. Nothing is ever declared from the first.
func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
	shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {

	// Every node, not only the placed ones. A machine that was never put on the private network
	// still runs modules, still holds claims, and still offers whatever it offers.
	nodes, err := inv.Nodes(ctx)
	if err != nil {
		return catalogue.World{}, err
	}
	places, err := inv.Overlays(ctx)
	if err != nil {
		return catalogue.World{}, err
	}
	siteOf := map[string]string{}
	for _, p := range places {
		siteOf[p.Name] = p.Site
	}
	// Which machines are actually on the private network, and what they are called there. Not
	// "has an address" — that was true of every placed machine and told you nothing about whether
	// anything could reach it. It is what resolved the module.
	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
	if err != nil {
		return catalogue.World{}, err
	}

	type candidate struct {
		node     catalogue.Node
		assigned []string
	}
	var others []candidate
	for _, n := range nodes {
		if n.Name == exclude {
			continue
		}
		theirs, err := inv.Assigned(ctx, n.Name)
		if err != nil || len(theirs) == 0 {
			continue
		}
		caps, _ := inv.ProfileOf(ctx, n.Name)
		others = append(others, candidate{
			catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
				At: onNetwork[n.Name]}, theirs})
	}

	offered := map[string][]catalogue.Provider{}
	for _, o := range others {
		got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
		if err != nil {
			// Their set does not resolve for some other reason. Not this node's problem to
			// report, and nothing of theirs is running, so it offers nothing.
			continue
		}
		for _, m := range got.Modules {
			for _, name := range m.OffersAt(catalogue.ScopeMesh) {
				// What that module says a consumer needs to know, with that node's settings on
				// it: a port somebody moved on the provider is a port its consumers must be told
				// about, and the two coming from different places is how they come to disagree.
				assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
				if err != nil {
					return catalogue.World{}, err
				}
				serves := catalogue.ServedOn(m, name, assigned)
				if len(serves) > 0 {
					layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
					if err != nil {
						return catalogue.World{}, err
					}
					serves, err = catalogue.Settle(serves, layers)
					if err != nil {
						return catalogue.World{}, err
					}
				}
				offered[name] = append(offered[name], catalogue.Provider{
					Node: o.node.Name, At: o.node.At, Serves: serves})
			}
		}
	}
	for k := range offered {
		sort.Slice(offered[k], func(i, j int) bool {
			return offered[k][i].Node < offered[k][j].Node
		})
	}

	world := catalogue.World{Offered: offered}
	for _, o := range others {
		got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
		if err != nil {
			continue
		}
		world.Held = append(world.Held, got.Claims...)
	}
	return world, nil
}

// declarationFor is everything a node would be sent.
//
// One place, because there were three and one of them was written before credentials existed and
// silently produced a declaration missing them — a difference between what `plan` showed and what
// `plan --json` handed to anything reading it.
func declarationFor(ctx context.Context, open *stores, node string,
	plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) {
	gens, err := generators(ctx, open)
	if err != nil {
		return nil, err
	}
	return declarationWith(ctx, open, node, plan, settings, gens, Reading)
}

// declarationWith is the same, for a caller that has already worked out the generators once and
// is about to use them for every node.
// Choosing says whether this composition may allocate what has not been allocated yet.
//
// **Asking what the mesh would send must not change what the mesh would send.** Composing a
// declaration assigns each module a machine port, and `status` composes one for every node to
// answer *is this machine running what I would send it* — so the question allocated, wrote, and
// contended with the very machine it was asking about. A status command that polls every two
// seconds while a node is applying is then two writers on the same rows, which is how it came to
// hang rather than answer.
//
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
// machine "waiting" means — the read needs no number to be right about that.
type Choosing bool

const (
	// Allocating is the send path: what is not assigned yet is assigned now and kept.
	Allocating Choosing = true
	// Reading is every question: what is assigned is used, and nothing is created.
	Reading Choosing = false
)

func declarationWith(ctx context.Context, open *stores, node string,
	plan catalogue.Resolution, settings catalogue.SettingsBy,
	gens map[string]catalogue.Generator, choosing Choosing) ([]map[string]any, error) {
	inv := open.inventory
	grants, err := grantsFor(ctx, open, node)
	if err != nil {
		return nil, err
	}
	// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
	//
	// **Assigned here rather than written by a module**, because a module is written once and
	// assigned anywhere: any number it picks is a guess about a machine it has never seen. Made
	// before the declaration is composed, because the container's mapping, the rule set and what a
	// consumer is told are all derived from it.
	// What this machine was already given, for a composition that may not allocate.
	already := map[string]map[int]int{}
	if choosing == Reading {
		held, err := inv.PortsFor(ctx, node)
		if err != nil {
			return nil, err
		}
		for _, a := range held {
			if already[a.Module] == nil {
				already[a.Module] = map[int]int{}
			}
			already[a.Module][a.Wanted] = a.Machine
		}
	}

	ports := map[string]map[int]int{}
	for _, m := range plan.Modules {
		for _, l := range m.Listens {
			// **Only a port the module actually publishes is the mesh's to move.** A container's
			// mapping is the thing that translates; without one the software binds what it binds,
			// and an assignment would not move the service — it would open the wrong number in the
			// rule set and leave the real one shut. Recorded either way, because this map means
			// *where this module's port is on this machine* and every reader of it needs that
			// answer whether or not the mesh was the one who chose it.
			where, mayAssign := m.MachineSide(l.Port)
			switch {
			case mayAssign && choosing == Allocating:
				at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
				if err != nil {
					return nil, fmt.Errorf(
						"%s needs %d reachable on %s and it could not be assigned: %w",
						m.Module, l.Port, node, err)
				}
				where = at.Machine
			case mayAssign:
				// Whatever was chosen last time, and nothing if there was no last time.
				if at, known := already[m.Module][l.Port]; known {
					where = at
				}
			}
			if ports[m.Module] == nil {
				ports[m.Module] = map[int]int{}
			}
			ports[m.Module][l.Port] = where
		}
	}

	// And each module's own secrets — a superuser password, an administrator, an account. Made
	// per node, so a module running on three machines has three.
	needed := map[string]map[string]string{}
	for _, m := range plan.Modules {
		for name := range m.OwnSecrets {
			// Minted on the send path and only read on every other. Making one is an insert, and
			// a question that writes is a question that can block against the machine it is about.
			var sealed string
			var err error
			if choosing == Allocating {
				sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
			} else {
				var held bool
				sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
				if err == nil && !held {
					// Never issued, so this machine cannot be running it. Left out rather than
					// invented: an empty string here would compose a declaration that differs
					// from what would be sent, and the comparison this feeds would then be
					// answering about a declaration nothing will ever push.
					continue
				}
			}
			if err != nil {
				return nil, err
			}
			if needed[m.Module] == nil {
				needed[m.Module] = map[string]string{}
			}
			needed[m.Module][name] = sealed
		}
	}
	// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
	// rather than stored: the node's key does not change, so signing again produces an equally
	// valid certificate and there is nothing to keep in step.
	var certificate, authority string
	for _, m := range plan.Modules {
		if m.Certificate == nil {
			continue
		}
		issued, meshCA, err := certificateFor(ctx, open, node)
		if err != nil {
			return nil, err
		}
		certificate, authority = issued, meshCA
		break
	}

	// And who else is on the private network, which is what a rule saying "from the mesh"
	// resolves to. Every node's address, including this one's: a machine reaching itself by its
	// own overlay address rather than by loopback is ordinary, and leaving it out would filter
	// the node's own traffic to itself with no rule naming why.
	private, err := onThePrivateNetwork(ctx, inv)
	if err != nil {
		return nil, err
	}

	// And every machine's name, so a container can reach one. The same set that writes the
	// machine's own hosts file — one reading, so a container and its machine cannot disagree
	// about where another machine is.
	names, err := namesInTheMesh(ctx, inv)
	if err != nil {
		return nil, err
	}

	return plan.Declaration(catalogue.Rendering{
		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
		Certificate: certificate, Authority: authority, Mesh: private, Names: names})
}

// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
// the machine and whether it is on the private network, `identity` holds the authority and the
// key that machine reported. The process holding both grants asks each for its part
// (novox/hq ADR 0008).
func certificateFor(ctx context.Context, open *stores, node string) (string, string, error) {
	inv := open.inventory
	ident, err := open.Identity(ctx)
	if err != nil {
		return "", "", err
	}

	record, err := inv.NodeByName(ctx, node)
	if err != nil {
		return "", "", err
	}
	serving, err := ident.ServingKeyOf(ctx, record.ID)
	if err != nil {
		return "", "", err
	}
	if serving == "" {
		// The machine joined before it had one, or never reported it. Said plainly, because the
		// remedy is on the machine and no amount of pushing from here will produce one.
		return "", "", fmt.Errorf(
			"%s wants a certificate and has never told the mesh what key it serves with; it "+
				"joins again to report one", node)
	}

	// The name it is certified for. Only a machine on the private network has one — a certificate
	// for a name nothing resolves is a certificate nothing can check.
	//
	// With the catalogue, not without it. Being on the private network is a conclusion about what
	// a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no
	// network — which refused every certificate the mesh was asked for, and said the machine was
	// not on a network it plainly was.
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return "", "", err
	}
	where, err := whereEveryoneIs(ctx, inv, shelf)
	if err != nil {
		return "", "", err
	}
	name := where[node]
	if name == "" {
		return "", "", fmt.Errorf(
			"%s wants a certificate and is not on the private network, so it has no name inside "+
				"the mesh to be certified for", node)
	}

	issued, err := ident.Certify(ctx, node, name, serving)
	if err != nil {
		return "", "", err
	}
	authority, err := ident.EstablishAuthority(ctx)
	if err != nil {
		return "", "", err
	}
	return issued, authority.Certificate, nil
}

// grantsFor is every credential this node must create, because something elsewhere uses it.
//
// The mirror of what a consumer is given, and the half that makes the credential real: a password
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
// the mesh hands over something it cannot itself use.
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
	inv := open.inventory
	issued, err := inv.SecretsFrom(ctx, node)
	if err != nil {
		return nil, err
	}

	// Where each consumer is, so a provider that must reach back to one does not have to know how
	// the mesh names machines.
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return nil, err
	}
	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
	if err != nil {
		return nil, err
	}

	// What each consumer actually asked for, taken from that machine's own resolution rather than
	// from a record beside it. A provider told to create a password and not what to create it for
	// can do nothing with it, and the name a consumer wants is the consumer's to say.
	out := make([]catalogue.Grant, 0, len(issued))
	for _, s := range issued {
		plan, settings, err := planFor(ctx, open, s.Consumer)
		if err != nil {
			// Their set does not resolve. Skipped rather than fatal: this node is not the place
			// to report another machine's problem, and a grant for something that is not going to
			// run would have the provider create a user nothing uses.
			continue
		}
		values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
		if err != nil {
			return nil, err
		}
		from := s.ConsumerModule
		if !asks {
			// That module no longer wants this. Left empty, which is what the declaration reads
			// as "nobody asks for it any more" — and is how a login is withdrawn rather than kept
			// working for ever after its consumer went away.
			from = ""
		}
		out = append(out, catalogue.Grant{
			Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
			From: from, Values: values, Sealed: s.ForProvider})
	}
	return out, nil
}

func planCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("plan", flag.ContinueOnError)
	// Because "one resource" does not tell you whether the settings landed. Being able to read
	// the file before it is sent is the difference between believing a merge worked and knowing.
	show := set.Bool("files", false, "print the files this node would be given")
	// The declaration exactly as the node would receive it. For handing to something else --
	// checking it against the host's own parser, most usefully, which is the only way to know
	// that what the control plane emits is what the host accepts.
	asJSON := set.Bool("json", false, "print the declaration this node would be sent")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if len(positionals) != 1 {
		return errors.New("plan <node> [--files] [--json]")
	}
	args = positionals
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()

	plan, settings, err := planFor(ctx, open, args[0])
	if err != nil {
		return err
	}
	if len(plan.Modules) == 0 {
		fmt.Printf("%s is assigned nothing\n", args[0])
		return nil
	}
	if *asJSON {
		resources, err := declarationFor(ctx, open, args[0], plan, settings)
		if err != nil {
			return err
		}
		body, err := json.MarshalIndent(
			map[string]any{"declaration": 1, "resources": resources}, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(body))
		return nil
	}

	fmt.Printf("%s would run:\n", args[0])
	for _, m := range plan.Modules {
		fmt.Printf("  %-20s %s\n", m.Module, plan.Because[m.Module])
	}
	for _, c := range plan.Claims {
		fmt.Printf("  holds %s, one per %s\n", c.Claim, c.Scope)
	}
	// What this machine depends on that is not on it. Worth saying out loud: it is the only part
	// of a node's set that stops working when a *different* machine goes away, and nothing else
	// in this output would have told anybody that.
	for _, n := range plan.Needs {
		fmt.Printf("  needs %s from %s, for %s\n", n.Name, n.From, n.For)
	}
	resources, err := declarationFor(ctx, open, args[0], plan, settings)
	if err != nil {
		return err
	}
	for module, layers := range settings {
		for _, layer := range layers {
			fmt.Printf("  %-20s settings from %s\n", module, layer.From)
		}
	}
	fmt.Printf("\n%d resource(s)\n", len(resources))

	if *show {
		for _, r := range resources {
			content, ok := r["content"].(string)
			if !ok {
				continue
			}
			fmt.Printf("\n--- %v  %v ---\n%s", r["id"], r["path"], content)
		}
	}
	return nil
}

// licencesFor is what this node can be answered with by record, and what it was put on.
//
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
// mesh is one, and a control plane that refused to plan because nobody had bought an API key
// would be unusable for the thing it already does.
func licencesFor(ctx context.Context, open *stores, node string) (
	map[string][]catalogue.Record, map[string]map[string]catalogue.Record, error) {

	held, err := open.Licences(ctx)
	if err != nil {
		return nil, nil, err
	}

	all, err := held.All(ctx)
	if err != nil {
		return nil, nil, err
	}
	if len(all) == 0 {
		return nil, nil, nil
	}

	offered := map[string][]catalogue.Record{}
	byName := map[string]catalogue.Record{}
	for _, one := range all {
		record := catalogue.Record{Name: one.Name, Serves: one.Serves}
		offered[licences.Provision] = append(offered[licences.Provision], record)
		byName[one.Name] = record
	}

	using := map[string]map[string]catalogue.Record{}
	for _, one := range all {
		holders, err := held.HoldersOf(ctx, one.Name)
		if err != nil {
			return nil, nil, err
		}
		for _, h := range holders {
			if h.Node != node {
				continue
			}
			if using[h.Module] == nil {
				using[h.Module] = map[string]catalogue.Record{}
			}
			using[h.Module][licences.Provision] = byName[one.Name]
		}
	}
	return offered, using, nil
}

// keyFor is the licence key sealed to one machine, for one module.
//
// **Empty is not an error here.** The mesh discarded the plaintext when it was supplied, so a
// holder recorded afterwards genuinely has no key — and the declaration refuses that by name,
// where the module and the path are both in view, rather than here.
func keyFor(ctx context.Context, open *stores, licence, node, module string) (string, error) {
	held, err := open.Licences(ctx)
	if err != nil {
		return "", err
	}
	return held.KeyFor(ctx, licence, node, module)
}

// portsOn is one module's assignments on one machine, by the port the software uses.
func portsOn(
	ctx context.Context, inv *inventory.Inventory, node, module string,
) (map[int]int, error) {
	all, err := inv.PortsFor(ctx, node)
	if err != nil {
		return nil, err
	}
	out := map[int]int{}
	for _, a := range all {
		if a.Module == module {
			out[a.Wanted] = a.Machine
		}
	}
	return out, nil
}