mesh-controller/internal/catalogue/resolve.go

package catalogue

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

// Resolving is turning "these modules are assigned here" into "this is what the node runs".
//
// It refuses rather than guesses, everywhere. novox/hq ADR 0009: a requirement with several
// answers is refused and named, because counting candidates has no surprising behaviour and a
// solver that picks has to be understood before its answer can be trusted.

// Node is what resolution needs to know about the machine.
type Node struct {
	Name string
	Site string
	// Capabilities the machine actually has, as its profile reported them. Only the present ones
	// — a capability that was looked for and not found is the same as one nobody looked for, as
	// far as deciding what may run here goes.
	Capabilities map[string]bool
	// At is this machine's own name on the private network, empty if it is not on one. Needed to
	// tell whether it can reach the node answering its requirements at all.
	At string
}

// World is what the rest of the mesh already has.
//
// Some requirements cannot be answered on the machine that has them — a database runs somewhere
// and is reached over the network — so resolving one node needs to know what the others offer.
type World struct {
	// Held is the claims already taken, for the scopes wider than one node.
	Held []Held
	// Offered is what other nodes provide at mesh scope, and everything needed to use it.
	Offered map[string][]Provider
	// Pinned is which node this machine was told to get a provision from, by name. Only consulted
	// when more than one node could answer -- a choice recorded before it was needed should not
	// start meaning something the day a second provider appears, and one recorded and then made
	// unnecessary should not quietly stop applying either.
	Pinned map[string]string
	// Licences is every provision answered by a **record rather than a node**, by provision name.
	//
	// novox/hq ADR 0024: a hosted model is on nobody's machine and is reached over the public
	// internet, so the rule that refuses two ends sharing no private network must not apply to
	// it. These are the candidates a refusal names.
	Licences map[string][]Record
	// Using is which record this node's modules were put on, keyed by module then provision.
	//
	// Per consumer, because that is the whole point: saying WHICH licence a given thing uses. Two
	// modules on one machine using different accounts is ordinary rather than a collision.
	Using map[string]map[string]Record

	// Unchecked takes brokered requirements on trust instead of refusing when nothing answers
	// them.
	//
	// For the first of two passes. Working out what a node offers the mesh needs that node
	// resolved, and resolving it may need what the mesh offers — so the first pass answers only
	// *what does each node offer*, and the second pass answers everything with that in hand. A
	// declaration is never built from an unchecked resolution.
	Unchecked bool
}

// Record is a provision answered by something the mesh holds rather than by a machine.
//
// The name is the operator's — *the personal account*, *the organisation's* — because the whole
// point is saying which one a consumer uses, and an anonymous credential hanging off a provider
// cannot be said (novox/hq ADR 0024).
type Record struct {
	// Name is what a person calls it, and what a consumer is put on.
	Name string
	// Serves is what a consumer must know that is not secret — a base URL, a model name.
	Serves map[string]any
}

// Provider is one node answering a mesh-scoped requirement.
type Provider struct {
	// Node is the machine.
	Node string
	// At is its name on the private network, or empty if it is not on one. The mesh's half of
	// the answer: a module can say it serves on port 5432, and only the mesh knows where.
	At string
	// Serves is what the providing module said a consumer needs to know, settled.
	Serves map[string]any
}

// Held is a claim somebody already has, used for the scopes wider than one node.
type Held struct {
	Claim  string
	Scope  string
	Node   string
	Module string
	Site   string
}

// Resolution is what a node should run, and why.
type Resolution struct {
	// Node is which machine this was resolved for, so a generator can be asked about it.
	Node string
	// At is this machine's own name on the private network, empty if it is not on one. Kept
	// because a consumer bound to something answered on this same machine still has to be told
	// where it is — the answer being local does not make the port guessable.
	At string

	// Modules in the order they were resolved: assigned first, then what they pulled in.
	Modules []Manifest
	// Because says why each module is here — assigned, or required by something.
	Because map[string]string
	// Claims is what this node's set holds, so wider scopes can be checked against it.
	Claims []Held
	// Needs is what this node's set gets from other nodes. Recorded rather than resolved away,
	// because it is where a credential will have to be handed back once there is a mechanism for
	// that, and because "what does this machine depend on that is not on it" has no other answer.
	Needs []Needed
}

// Needed is one thing this node's set takes from elsewhere in the mesh.
type Needed struct {
	// Name is the provision, as required.
	Name string
	// From is the node providing it.
	From string
	// At is where that node is on the private network.
	At string
	// Serves is what the providing module said a consumer needs to know.
	Serves map[string]any
	// ByRecord means this was answered by something the mesh holds rather than by a machine, so
	// there is no node to reach and no private network to share. Its credential comes from
	// wherever that record's does, not from the pair-wise secret two machines share.
	ByRecord bool
	// Sealed is the credential, closed to this node. Filled in after resolving, because whose
	// credential it is only becomes answerable once which node answers has been settled.
	Sealed string
	// For is the module that wanted it.
	For string
}

// Refusal is why a set of assignments cannot become a declaration.
//
// Every reason at once rather than the first, and each says what to do about it. A person
// resolving these fixes them in one pass or in four.
type Refusal struct{ Problems []string }

func (r *Refusal) Error() string {
	return "these assignments cannot be applied:\n  - " + strings.Join(r.Problems, "\n  - ")
}

// ErrAmbiguous is returned inside a Refusal when a requirement has more than one answer.
var ErrAmbiguous = errors.New("more than one module provides that")

// Resolve works out everything a node runs, from what was assigned to it.
//
// The catalogue is every module the mesh knows about; assigned is what a person put on this node.
// What comes back is the closure — assigned modules plus everything they require — or a refusal
// naming every reason it could not be closed.
func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world World) (Resolution, error) {
	elsewhere := world.Held
	var problems []string

	// Which names are answered from elsewhere in the mesh rather than from this machine. A
	// property of the name, not of each provider: two modules disagreeing about whether a
	// database is local would make the same requirement mean different things depending on which
	// one happened to answer it.
	brokered := map[string]bool{}
	local := map[string]bool{}
	for _, m := range catalogue {
		for _, o := range m.Provides {
			if o.At() == ScopeMesh {
				brokered[o.Name] = true
				continue
			}
			local[o.Name] = true
		}
	}
	for want := range brokered {
		if local[want] {
			problems = append(problems, fmt.Sprintf(
				"the catalogue disagrees about %q: some modules provide it here and others from "+
					"anywhere in the mesh, so the same requirement would mean two things", want))
		}
	}

	// What each name can be satisfied by. Built once from the whole catalogue, because "how many
	// modules provide this" is the question the whole rule turns on.
	offers := map[string][]string{}
	for _, m := range catalogue {
		for _, o := range m.Offers() {
			offers[o] = append(offers[o], m.Module)
		}
	}
	for k := range offers {
		sort.Strings(offers[k])
	}

	chosen := map[string]bool{}
	because := map[string]string{}
	var order []string
	var needs []Needed

	// What the set already offers, which is the first thing a requirement is checked against.
	//
	// Without this, assigning zsh does not satisfy something that requires a shell: the
	// requirement is counted against the catalogue, three modules provide it, and the answer is
	// still "choose one" after somebody has chosen one. That makes the remedy useless, and it is
	// how this read when first used.
	satisfied := map[string]bool{}

	// Everything a person assigned goes in first. Those are choices already made, and a
	// requirement one of them answers is not a choice to put back to anybody.
	queue := append([]string{}, assigned...)
	for _, a := range assigned {
		because[a] = "assigned"
		if m, known := catalogue[a]; known {
			for _, o := range m.Offers() {
				satisfied[o] = true
			}
		}
	}

	// What has already been complained about. A requirement can be wanted by several modules at
	// once, and saying the same thing twice makes a person hunt for the difference between two
	// identical lines before realising there is none.
	reported := map[string]bool{}

	for len(queue) > 0 {
		want := queue[0]
		queue = queue[1:]
		if chosen[want] || reported[want] {
			continue
		}
		// Already answered by something in the set. This is the case that makes assigning zsh do
		// what a person meant by it.
		//
		// **Except when the thing answering it grants a credential** (novox/hq 04-ISSUES/021). A
		// shell answered here needs nothing more; a database answered here still needs a
		// password, because the consumer reaches it over TCP from its own container and the
		// database asks exactly as it would from another machine. **The machine stops being a
		// trust boundary the moment both ends are containers**, and treating it as one gave the
		// commonest arrangement of all — a service and its database on one node — the weakest
		// handling, silently.
		if satisfied[want] && !isModule(catalogue, want) {
			if brokered[want] {
				// Answered here, and still a need: the provider is this node.
				needs = append(needs, Needed{
					Name: want, From: node.Name, At: node.At,
					Serves: servedHere(catalogue, chosen, want), For: because[want]})
			}
			continue
		}

		// Answered from elsewhere in the mesh, if it is that kind of name. **Never installed
		// here.** Choosing a machine to put a database on is a decision with consequences
		// nobody would want made silently by something resolving a web application.
		if brokered[want] {
			reported[want] = true
			where := world.Offered[want]
			names := make([]string, 0, len(where))
			for _, p := range where {
				names = append(names, p.Node)
			}
			sort.Strings(names)
			take := func(p Provider) {
				if node.At != "" && p.At == "" || node.At == "" && p.At != "" || node.At == "" && p.At == "" {
					// One of them is not on the private network, so there is no path between
					// them. Said here rather than discovered as a connection timing out on a
					// machine that the mesh reported as configured.
					problems = append(problems, fmt.Sprintf(
						"%s needs %q from %s, and they are not both on the private network — "+
							"assign %s to whichever is missing it",
						node.Name, want, p.Node, meshNetwork))
					return
				}
				needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
					Serves: p.Serves, For: because[want]})
			}
			switch {
			case world.Unchecked:
				// First pass. Whether anything answers this is exactly the question this pass
				// exists to make answerable, so it is not asked here.
			case len(where) == 0:
				remedy := "and nothing in the catalogue could"
				if answers := offers[want]; len(answers) == 1 {
					remedy = "— assign " + answers[0] + " to a node"
				} else if len(answers) > 1 {
					remedy = "— assign one of these to a node: " + strings.Join(answers, ", ")
				}
				problems = append(problems, fmt.Sprintf(
					"nothing in this mesh provides %q, wanted by %s %s",
					want, because[want], remedy))
			case len(where) == 1:
				if chosenNode, pinned := world.Pinned[want]; pinned && chosenNode != where[0].Node {
					// One provider, and it is not the one this machine was told to use. Silently
					// using the other would be the mesh overruling a choice somebody made.
					problems = append(problems, fmt.Sprintf(
						"%s was told to get %q from %s, and only %s provides it",
						node.Name, want, chosenNode, where[0].Node))
					break
				}
				take(where[0])
			default:
				chosenNode, pinned := world.Pinned[want]
				if !pinned {
					problems = append(problems, fmt.Sprintf(
						"%d nodes provide %q, wanted by %s — say which with `pin %s %s <node>`: %s",
						len(where), want, because[want], node.Name, want,
						strings.Join(names, ", ")))
					break
				}
				var chosen *Provider
				for i, w := range where {
					if w.Node == chosenNode {
						chosen = &where[i]
					}
				}
				if chosen == nil {
					// Pointed at a machine that does not answer this. Refused rather than
					// falling back to another: a fallback would quietly move somebody's data to
					// a machine they did not choose, which is the whole reason this is asked.
					problems = append(problems, fmt.Sprintf(
						"%s was told to get %q from %s, and %s does not provide it — these do: %s",
						node.Name, want, chosenNode, chosenNode, strings.Join(names, ", ")))
					break
				}
				take(*chosen)
			}
			continue
		}

		candidates := offers[want]
		switch len(candidates) {
		case 0:
			if len(world.Licences[want]) > 0 {
				// Answered by a record rather than by a module, and the post-pass below settles
				// which one. Left alone here: the two questions a refusal must answer — *is
				// there anything* and *which one* — have different remedies, and answering the
				// first wrongly would send somebody looking for a module to install.
				reported[want] = true
				continue
			}
			if world.Unchecked {
				// The first pass, which exists only to answer *what does this node offer*.
				//
				// **Refusing here makes the machine vanish** rather than reporting a problem: the
				// caller takes a failed resolution to mean it learned nothing about this node, so
				// one unanswerable requirement on one machine removes that machine from the
				// private network as far as every OTHER machine is concerned — and they are then
				// told, wrongly, that the two of them share no network.
				//
				// That is a wrong answer about a machine nobody asked about, caused by a fault on
				// a third. The second pass refuses it properly, where the question is being asked.
				reported[want] = true
				continue
			}
			reported[want] = true
			problems = append(problems, fmt.Sprintf(
				"nothing provides %q, wanted by %s", want, because[want]))
			continue
		case 1:
			// No choice to make, so none is made. This is the case that lets `install i3` bring
			// in xorg without anybody being asked anything.
		default:
			reported[want] = true
			problems = append(problems, fmt.Sprintf(
				"%q is wanted by %s and %d modules provide it — choose one and assign it: %s",
				want, because[want], len(candidates), strings.Join(candidates, ", ")))
			continue
		}

		m := catalogue[candidates[0]]
		if chosen[m.Module] {
			continue
		}
		chosen[m.Module] = true
		order = append(order, m.Module)
		for _, o := range m.Offers() {
			satisfied[o] = true
		}
		if _, ok := because[m.Module]; !ok {
			because[m.Module] = fmt.Sprintf("required by %s", because[want])
		}

		for _, r := range m.Wants() {
			if _, ok := because[r]; !ok {
				because[r] = m.Module
			}
			queue = append(queue, r)
		}
	}

	// One need per module that wants it, rather than one per name (novox/hq 04-ISSUES/022).
	//
	// **A consumer is a module on a machine, not a machine.** The walk above is a work-list over
	// names, so a requirement three modules share is visited once and produced one need, carrying
	// whichever module happened to mention it first. Everything downstream inherited that: one
	// credential, named after a node, and the other two consumers given nothing at all — a
	// service that resolves cleanly and then cannot authenticate, which is the exact shape of
	// 021.
	//
	// The record pass below already gets this right and says so. It is the same rule.
	needs = perConsumer(needs, order, catalogue)

	// What is answered by a record rather than by a machine.
	//
	// A post-pass, deliberately: nothing about it depends on the order requirements were walked
	// in, and putting it in the queue would mean the reachability rule — which must not apply
	// here — sitting one branch away from a case it would be wrong for.
	//
	// **Refused when the consumer has not said which.** ADR 0024 warns this will be felt: a mesh
	// holding three ways to reach a model refuses every consumer that has not chosen, which is
	// correct and is a great deal of saying-which the first time. So the refusal names the
	// candidates and the exact command, because being right is not the same as being usable.
	for _, name := range order {
		m := catalogue[name]
		for _, want := range m.Wants() {
			offered, byRecord := world.Licences[want]
			if !byRecord || len(offered) == 0 {
				continue
			}
			if satisfied[want] {
				// Something in this node's own set answers it -- a model the mesh runs itself,
				// most obviously. A record is not consulted when there is a local answer.
				continue
			}
			using, said := world.Using[m.Module][want]
			if !said {
				if world.Unchecked {
					continue
				}
				names := make([]string, 0, len(offered))
				for _, r := range offered {
					names = append(names, r.Name)
				}
				sort.Strings(names)
				problems = append(problems, fmt.Sprintf(
					"%s on %s needs %q and has not been told which one to use — say which with "+
						"`licence use <name> %s %s`: %s",
					m.Module, node.Name, want, node.Name, m.Module, strings.Join(names, ", ")))
				continue
			}
			needs = append(needs, Needed{
				Name: want, From: using.Name, Serves: using.Serves, ByRecord: true,
				// The module that required it, not whatever first mentioned the name: the key is
				// sealed per consumer, and a consumer here is a module on a machine.
				For: m.Module,
			})
		}
	}

	resolution := Resolution{Node: node.Name, At: node.At, Because: because, Needs: needs}
	for _, n := range providersFirst(order, catalogue) {
		resolution.Modules = append(resolution.Modules, catalogue[n])
	}

	problems = append(problems, checkCapabilities(resolution.Modules, node)...)
	claims, claimProblems := checkClaims(resolution.Modules, node, elsewhere)
	problems = append(problems, claimProblems...)
	problems = append(problems, checkResources(resolution.Modules)...)
	resolution.Claims = claims

	if len(problems) > 0 {
		sort.Strings(problems)
		return Resolution{}, &Refusal{Problems: problems}
	}
	return resolution, nil
}

// servedHere is what a provider in this node's own set says a consumer needs to know.
//
// The same facts a provider elsewhere would have contributed through the world, taken from the
// module directly because a provider on this machine never passes through it.
func servedHere(catalogue map[string]Manifest, chosen map[string]bool, want string) map[string]any {
	for name, m := range catalogue {
		if !chosen[name] {
			continue
		}
		if _, ok := m.Serves[want]; ok {
			// Without assignments: this is resolution, which runs before a machine's ports are
			// known. The declaration fills the machine's own in afterwards, where it has them.
			return ServedOn(m, want, nil)
		}
	}
	return nil
}

// isModule reports whether a name is a module in its own right rather than only something
// modules provide.
//
// A requirement naming a module is not satisfied by something else providing that name: `i3`
// requires `xorg` and means xorg, not "anything calling itself a display server".
func isModule(catalogue map[string]Manifest, want string) bool {
	_, ok := catalogue[want]
	return ok
}

// checkCapabilities refuses a module the machine cannot run.
//
// Said as a fact about the machine rather than about the module, because that is what it is and
// because nothing can be installed to change it.
func checkCapabilities(modules []Manifest, node Node) []string {
	var problems []string
	for _, m := range modules {
		for _, c := range m.Capabilities {
			if !node.Capabilities[c] {
				problems = append(problems, fmt.Sprintf(
					"%s needs the capability %q and %s does not have it — this is the wrong "+
						"machine, not a missing module", m.Module, c, node.Name))
			}
		}
	}
	return problems
}

// checkClaims refuses two modules holding one singular thing.
//
// Within this node's own set, and against what is already held elsewhere for the wider scopes. A
// claim at mesh scope is the same idea as the mesh's one hub, said once instead of hard-coded.
func checkClaims(modules []Manifest, node Node, elsewhere []Held) ([]Held, []string) {
	var problems []string
	var held []Held

	byScope := map[string]map[string]string{} // scope → claim → module
	for _, m := range modules {
		for _, c := range m.Claims {
			scope := c.At()
			if byScope[scope] == nil {
				byScope[scope] = map[string]string{}
			}
			if other, taken := byScope[scope][c.Name]; taken {
				problems = append(problems, fmt.Sprintf(
					"%s and %s both claim %q, and only one thing may hold it per %s",
					other, m.Module, c.Name, scope))
				continue
			}
			byScope[scope][c.Name] = m.Module
			held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
				Module: m.Module, Site: node.Site})
		}
	}

	// And against the rest of the mesh, for the scopes that reach past this machine.
	for _, h := range held {
		for _, e := range elsewhere {
			if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
				continue
			}
			switch h.Scope {
			case ScopeMesh:
				problems = append(problems, fmt.Sprintf(
					"%s on %s claims %q, which %s on %s already holds — one per mesh",
					h.Module, node.Name, h.Claim, e.Module, e.Node))
			case ScopeSite:
				if node.Site != "" && node.Site == e.Site {
					problems = append(problems, fmt.Sprintf(
						"%s on %s claims %q, which %s on %s already holds at %s — one per site",
						h.Module, node.Name, h.Claim, e.Module, e.Node, node.Site))
				}
			}
		}
	}
	return held, problems
}

// checkResources refuses two modules writing the same thing.
//
// This costs no manifest field: the mesh already holds every resource of every module, so two
// declaring one path or one unit are visible without either having to know about the other. A
// declared claim is only for the abstract conflicts nothing in the resources reveals.
func checkResources(modules []Manifest) []string {
	var problems []string
	owner := map[string]string{}

	for _, m := range modules {
		for _, r := range m.Resources {
			for _, field := range []string{"path", "unit", "name", "package"} {
				value, ok := r[field].(string)
				if !ok || value == "" {
					continue
				}
				key := field + " " + value
				if other, taken := owner[key]; taken && other != m.Module {
					problems = append(problems, fmt.Sprintf(
						"%s and %s both declare the %s %q", other, m.Module, field, value))
				}
				owner[key] = m.Module
			}
		}
	}
	return problems
}

// Offers is a list of node-scoped provisions, which is what nearly everything is.
func Offers(names ...string) []Offer {
	out := make([]Offer, 0, len(names))
	for _, n := range names {
		out = append(out, Offer{Name: n})
	}
	return out
}

// FromAnywhere is a provision answered by whichever node in the mesh runs it.
func FromAnywhere(names ...string) []Offer {
	out := make([]Offer, 0, len(names))
	for _, n := range names {
		out = append(out, Offer{Name: n, Scope: ScopeMesh})
	}
	return out
}

// meshNetwork is what to assign to a machine that needs to reach another one.
//
// A string here rather than an import, because the private network is a module the control plane
// ships and this package must not depend on the thing it resolves. The name being wrong would
// show up as a refusal naming a module nobody can assign, which a test checks.
const meshNetwork = "networking"

// providersFirst orders a node's modules so that what answers a requirement comes before what
// asked for it.
//
// **The host does not sort** (novox/hq ADR 0005), so the order written here is the order a machine
// applies. Selection walks outward from what was assigned, which puts a consumer before the thing
// it pulled in — and a module whose service reads a file another module writes then starts before
// the file exists.
//
// It fails, and the next reconcile fixes it. That is the worst shape a fault can take: what gets
// remembered is that it works, and nobody looks again
// ([04-ISSUES/013](novox/hq)). This is the same fault one level up from where that was found.
//
// **Stable where nothing requires anything.** Modules with no relation keep the order selection
// gave them, because that order is meaningful — assigned first, then what they pulled in — and a
// sort that reshuffled unrelated modules would make every declaration's diff unreadable.
func providersFirst(order []string, shelf map[string]Manifest) []string {
	// Who offers what, among the modules actually chosen.
	offeredBy := map[string]string{}
	for _, name := range order {
		for _, offered := range shelf[name].Offers() {
			if _, taken := offeredBy[offered]; !taken {
				offeredBy[offered] = name
			}
		}
	}

	needs := map[string][]string{}
	for _, name := range order {
		for _, want := range shelf[name].Wants() {
			provider, known := offeredBy[want]
			if !known || provider == name {
				continue
			}
			needs[name] = append(needs[name], provider)
		}
	}

	var out []string
	placed := map[string]bool{}
	var place func(string, map[string]bool)
	place = func(name string, visiting map[string]bool) {
		if placed[name] {
			return
		}
		if visiting[name] {
			// Two modules requiring each other. Left in the order selection gave them rather
			// than refused: a cycle here is not a machine that cannot work — both are applied,
			// and one of them starts before the other is ready and settles on the next pass.
			// Refusing would make a pair of modules that cooperate impossible to assign.
			return
		}
		visiting[name] = true
		for _, provider := range needs[name] {
			place(provider, visiting)
		}
		delete(visiting, name)
		placed[name] = true
		out = append(out, name)
	}
	for _, name := range order {
		place(name, map[string]bool{})
	}
	return out
}

// perConsumer turns one need per provision into one need per module that wants it.
//
// Order follows the resolved modules rather than a map, so the same set always produces the same
// needs — a declaration whose contents move for no reason makes every push look like a change.
//
// A need nothing in the set wants is kept as it is rather than dropped. That should not happen;
// if it does, the honest outcome is an extra credential nobody reads, not a consumer silently
// losing the one it depends on.
func perConsumer(needs []Needed, order []string, catalogue map[string]Manifest) []Needed {
	out := make([]Needed, 0, len(needs))
	for _, n := range needs {
		var wanted bool
		for _, name := range order {
			m, known := catalogue[name]
			if !known {
				continue
			}
			for _, want := range m.Wants() {
				if want != n.Name {
					continue
				}
				copied := n
				copied.For = m.Module
				out = append(out, copied)
				wanted = true
				break
			}
		}
		if !wanted {
			out = append(out, n)
		}
	}
	return out
}