mesh-controller/internal/catalogue/manifest.go

// Package catalogue is what modules are, and what a node gets when it is assigned some.
//
// novox/hq ADR 0009: everything is a module, a module declares what it provides and requires,
// and a module declares what it claims. This turns a set of assignments into the one declaration
// a node is sent — which is the first thing the control plane decides rather than relays.
package catalogue

import (
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

// Scopes a claim can have.
//
// Not everything singular is singular per machine: a seat is one per node, a DHCP server is one
// per segment, and the hub is one per mesh. Scope says which, and it is the same idea the mesh
// already enforces by hand for the hub.
const (
	ScopeNode = "node"
	ScopeSite = "site"
	ScopeMesh = "mesh"
)

// name is what a module, a provision or a claim may be called.
//
// Constrained because these become resource identities, permission patterns and error messages,
// and a name that is valid in one and not the others is a fault found late.
var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(\.[a-z0-9][a-z0-9-]*)*$`)

// Claim is a singular resource a module takes over.
type Claim struct {
	Name string `json:"name"`
	// Scope defaults to the node, which is where nearly everything singular is singular.
	Scope string `json:"scope,omitempty"`
}

// At is this claim's scope, with the default applied.
func (c Claim) At() string {
	if c.Scope == "" {
		return ScopeNode
	}
	return c.Scope
}

// Offer is something a module provides, and where the answer to it may live.
//
// **The distinction this exists for:** a shell, a display server and a private network have to be
// on the machine that needs them. A database, an object store and an identity provider do not —
// they run somewhere in the mesh and are reached over it. Treating the second as the first
// installs PostgreSQL on every machine that runs a web application, which is what happened until
// this field existed.
//
// Written as a bare string in the ordinary case, because nearly everything is node-scoped and
// making every manifest say so would bury the few that are not:
//
//	"provides": ["shell"]
//	"provides": [{"name": "database", "scope": "mesh"}]
type Offer struct {
	Name string `json:"name"`
	// Scope defaults to the node, which is where most things must be to be usable.
	Scope string `json:"scope,omitempty"`
}

// At is this offer's scope, with the default applied.
func (o Offer) At() string {
	if o.Scope == "" {
		return ScopeNode
	}
	return o.Scope
}

// UnmarshalJSON accepts a plain name as well as an object.
func (o *Offer) UnmarshalJSON(raw []byte) error {
	var plain string
	if err := json.Unmarshal(raw, &plain); err == nil {
		o.Name, o.Scope = plain, ""
		return nil
	}
	var full struct {
		Name  string `json:"name"`
		Scope string `json:"scope,omitempty"`
	}
	if err := json.Unmarshal(raw, &full); err != nil {
		return fmt.Errorf("a provided name is either a string or {name, scope}: %w", err)
	}
	o.Name, o.Scope = full.Name, full.Scope
	return nil
}

// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
// went through the mesh comes out looking like the one that went in.
func (o Offer) MarshalJSON() ([]byte, error) {
	if o.Scope == "" {
		return json.Marshal(o.Name)
	}
	return json.Marshal(struct {
		Name  string `json:"name"`
		Scope string `json:"scope"`
	}{o.Name, o.Scope})
}

// Manifest is everything a module says about itself.
type Manifest struct {
	Module  string `json:"module"`
	Version string `json:"version,omitempty"`

	// Provides are the names other modules may require. A module always provides its own name;
	// this is for the rest — `zsh` provides `shell`, `xorg` provides `display-server`.
	Provides []Offer `json:"provides,omitempty"`

	// Requires are names that must be provided by something assigned to the same node.
	Requires []string `json:"requires,omitempty"`

	// Claims are singular resources. Two modules claiming one thing within a scope cannot both
	// be assigned there — which is how exclusivity is expressed, rather than as a list of rivals
	// that every new module would force its predecessors to update.
	Claims []Claim `json:"claims,omitempty"`

	// Capabilities the machine must have. A different field from Requires because the remedy
	// differs: a missing module can be assigned, and a missing capability means the wrong
	// machine.
	Capabilities []string `json:"capabilities,omitempty"`

	// Resources are what this module puts on a node, in the host's own vocabulary.
	Resources []map[string]any `json:"resources,omitempty"`

	// Computed names something in the control plane that works this module's resources out per
	// node, instead of them being fixed here.
	//
	// Because some files cannot be written in advance. A machine's peer list on the private
	// network is derived from every other machine, so it differs on each one and changes when any
	// of them changes — there is nothing to put in a manifest.
	//
	// Being a module anyway is the point: it is assigned like anything else, so a machine that
	// should not be on the private network simply is not given it, and the network is worked out
	// over the machines that have it. Before this, connectivity was code beside the module system
	// doing the same job, and every machine with an address was on the network whether or not
	// anybody wanted it there.
	Computed string `json:"computed,omitempty"`

	// Contributes is what this module tells whatever answers a requirement.
	//
	// The other half of an edge. `requires` says a thing must be there; this says what to do with
	// it — a web application requiring a reverse proxy has to say *which name, which port*, and
	// until now there was nowhere to put that. Every module that needed it was reduced to
	// reaching into the control plane's database directly, which is how two of them came to hold
	// a credential to it permanently.
	//
	// Keyed by the requirement, because that is what the contribution is *about*. Contributing to
	// something is requiring it: asking to be published means a publisher must exist, and a
	// module that had to say both would eventually say one.
	Contributes map[string]map[string]any `json:"contributes,omitempty"`

	// Receives is where this module wants its consumers' contributions written, per requirement
	// it provides.
	//
	// A file, in the mesh's own shape, replaced whenever the set changes. **The control plane
	// does not know what a reverse proxy is** and does not write one's configuration — it
	// delivers the facts, and the module turns them into whatever it runs. That boundary is why
	// swapping the proxy does not touch a single module that publishes through it.
	Receives map[string]string `json:"receives,omitempty"`

	// Serves is what a consumer needs to know in order to use something this module provides — a
	// port, a path, a realm. The module's half of the answer; the mesh adds the other half, which
	// is *which machine* and *where it is on the private network*.
	//
	// It does not carry a credential and cannot: a manifest is the same on every mesh, and a
	// secret is the one thing that must not be.
	Serves map[string]map[string]any `json:"serves,omitempty"`

	// Build says how this module's artifacts are produced from its source.
	//
	// The manifest in a repository names artifacts; the manifest the mesh holds names digests.
	// **They are not the same document**, and that is deliberate: a digest is not knowable until
	// something is built, and a repository that carried one would be a repository whose file is
	// wrong the moment anybody edits anything.
	Build *Build `json:"build,omitempty"`

	// Binds is where this module wants to be told about something it requires, per requirement.
	//
	// Because "this machine needs a database from the anchor" is useless to the program that
	// needs it unless the program is told. A file, like everything else — the host writes files
	// and knows nothing about provisions, which is what keeps this from needing anything new
	// down there.
	Binds map[string]string `json:"binds,omitempty"`

	// Secrets is where this module wants the credential for something it requires, per
	// requirement. The file holds the value and nothing else, so a program can read it without
	// parsing anything.
	//
	// **Its own file, separate from Binds, because the mesh cannot compose a document containing
	// it.** The value was sealed to this node when it was made and the plaintext discarded — so
	// there is nothing to interpolate into a larger file, and that is the property worth keeping
	// rather than an inconvenience to work around. It also means the readable half stays readable
	// in the declaration, and the secret half changes only when the secret does, which is what
	// makes `restart-on` precise.
	Secrets map[string]string `json:"secrets,omitempty"`

	// Needs is a secret this module needs for itself, and where to put it.
	//
	// Not tied to a consumer. A database has a superuser password, a broker has an administrator,
	// a registry has an account — each is a secret the module needs in order to be itself, and
	// none of them is *for* anybody. Keyed by a name of the module's choosing, valued by the file
	// it lands in.
	//
	// **Generated per node and sealed to it**, like everything else the mesh hands out, so a
	// module running on three machines has three passwords and the mesh can read none of them. A
	// manifest carrying one instead would put the same secret on every machine that ever runs the
	// module, in a file anybody can read, for ever.
	Needs map[string]string `json:"needs,omitempty"`

	// Certificate is where this module wants a certificate for its machine's name inside the
	// mesh, and where the key that goes with it can be found.
	//
	// **The key is named, not delivered.** The node generated it at enrolment and keeps it; the
	// mesh only ever signs the public half. So what arrives is a certificate, which is public,
	// and a path to a file the machine already has.
	//
	// Two authorities are kept apart on purpose (novox/hq 08-connectivity): this is the mesh's,
	// for names only the mesh knows. A name the outside world reaches is a different authority
	// and a different problem.
	Certificate *Certificate `json:"certificate,omitempty"`

	// Grants is a directory this module wants the credentials of its consumers written into, per
	// provision it offers — one file per consumer, named for it, holding the value alone.
	//
	// A directory rather than one document for the same reason as above: each value is sealed
	// separately and the mesh cannot open any of them to build a list.
	Grants map[string]string `json:"grants,omitempty"`
}

// Build says how to produce this module's artifacts from its source.
//
// **Absent means nothing is built.** A module can be entirely configuration — a shell's rc file,
// a set of firewall rules — and having to declare an empty build for it would be a field that
// exists to be left blank.
type Build struct {
	// Artifacts are what the source produces, each named so a resource can refer to it before
	// anybody knows its digest.
	Artifacts []Artifact `json:"artifacts,omitempty"`
}

// Artifact is one thing built from a module's source.
type Artifact struct {
	// Name is how resources refer to it. Local to the module.
	Name string `json:"name"`
	// Kind is "image" or "archive".
	Kind string `json:"kind"`
	// From is what it is built from, relative to the repository root: a Dockerfile for an image,
	// a directory for an archive.
	From string `json:"from"`
}

// Kinds an artifact may be.
const (
	// ArtifactImage is built from a Dockerfile in this repository.
	ArtifactImage = "image"
	// ArtifactArchive is a directory in this repository, packed.
	ArtifactArchive = "archive"
	// ArtifactUpstream is an image somebody else built, mirrored into the mesh's own registry and
	// pinned by the digest it lands with.
	//
	// **Because a module usually runs software it did not write.** A database module ships
	// configuration and a provisioner and does not build a database. It could name the upstream
	// reference directly, and then every machine needs a route to a public registry and the
	// reference is a tag somebody else can move — which is what pinning exists to prevent
	// (novox/hq ADR 0006).
	//
	// Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into
	// the registry a first node pulls from. This makes that a thing a module can say.
	ArtifactUpstream = "upstream"
)

// Certificate says where a module wants what the mesh issued for its machine.
type Certificate struct {
	// Into is where the certificate is written.
	Into string `json:"into"`
	// Authority is where the mesh's own certificate is written, so something connecting to this
	// machine can be told what to believe. Optional: a module that only serves does not need it.
	Authority string `json:"authority,omitempty"`
}

// CertificateID and AuthorityID are the resource identities of what the mesh issued.
func CertificateID() string { return "certificate" }
func AuthorityID() string   { return "certificate-authority" }

// NeedID is the resource identity of the file a module's own secret lands in.
func NeedID(name string) string { return "needs-" + name }

// SecretID is the resource identity of the file a module is given a credential in.
func SecretID(requirement string) string { return "secret-" + requirement }

// GrantID is the resource identity of one consumer's credential on the providing machine.
func GrantID(provision, consumer string) string { return "grant-" + provision + "-" + consumer }

// BoundID is the resource identity of the file a module is told about a provision in.
func BoundID(requirement string) string { return "bound-" + requirement }

// Wants is everything that must be provided on the same node: what this module requires, and what
// it contributes to.
func (m Manifest) Wants() []string {
	out := append([]string{}, m.Requires...)
	for to := range m.Contributes {
		var already bool
		for _, r := range m.Requires {
			if r == to {
				already = true
			}
		}
		if !already {
			out = append(out, to)
		}
	}
	sort.Strings(out)
	return out
}

// ReceivedID is the resource identity of the file a provider is given its contributions in.
//
// Named rather than positional so a module can point `restart-on` at it: a proxy that got a new
// route and did not reload is a route that silently does not work, which is the same fault the
// overlay had when a peer list changed under a running interface.
func ReceivedID(requirement string) string { return "received-" + requirement }

// ParseManifest reads a module manifest, refusing anything it cannot act on.
//
// Every problem is reported rather than the first, because somebody writing a manifest fixes
// them in one pass or in four.
func ParseManifest(raw []byte) (Manifest, error) {
	var m Manifest
	if err := json.Unmarshal(raw, &m); err != nil {
		return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err)
	}

	var problems []string
	if !name.MatchString(m.Module) {
		problems = append(problems, fmt.Sprintf(
			"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
	}
	for _, offer := range m.Provides {
		p := offer.Name
		if !name.MatchString(p) {
			problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
		}
		if s := offer.At(); s != ScopeNode && s != ScopeMesh {
			// Site scope is meaningful for a claim — one DHCP server per segment — and is not
			// yet meaningful for a provision, because nothing knows how to reach "the one at my
			// site". Refused rather than silently treated as mesh-wide.
			problems = append(problems, fmt.Sprintf(
				"%s provides %q at scope %q; a provision is %q or %q",
				m.Module, p, s, ScopeNode, ScopeMesh))
		}
		if p == m.Module {
			// Harmless and worth saying: a module always provides its own name, so writing it
			// suggests the author expected it not to.
			problems = append(problems, fmt.Sprintf(
				"%s provides its own name already; listing it says nothing", m.Module))
		}
	}
	for _, r := range m.Requires {
		if !name.MatchString(r) {
			problems = append(problems, fmt.Sprintf("%q is not a usable name to require", r))
		}
		if r == m.Module {
			problems = append(problems, fmt.Sprintf("%s requires itself", m.Module))
		}
	}
	for _, c := range m.Claims {
		if !name.MatchString(c.Name) {
			problems = append(problems, fmt.Sprintf("%q is not a usable claim name", c.Name))
		}
		switch c.At() {
		case ScopeNode, ScopeSite, ScopeMesh:
		default:
			problems = append(problems, fmt.Sprintf(
				"%s claims %s at scope %q; a claim is held per node, per site or per mesh",
				m.Module, c.Name, c.Scope))
		}
	}
	if m.Computed != "" && len(m.Resources) > 0 {
		// One or the other. A module that both ships files and has them computed would leave
		// nobody able to say where a given file came from.
		problems = append(problems, fmt.Sprintf(
			"%s has resources of its own and says they are computed by %q; it is one or the other",
			m.Module, m.Computed))
	}
	for to, values := range m.Contributes {
		if !name.MatchString(to) {
			problems = append(problems, fmt.Sprintf("%q is not a usable name to contribute to", to))
		}
		if len(values) == 0 {
			// An empty contribution is either a mistake or a requirement written the long way
			// round, and both are better said plainly.
			problems = append(problems, fmt.Sprintf(
				"%s contributes nothing to %q; if it only needs one, require it", m.Module, to))
		}
	}
	problems = append(problems, m.Build.problems(m.Module)...)
	for to := range m.Serves {
		var offered bool
		for _, o := range m.Offers() {
			if o == to {
				offered = true
			}
		}
		if !offered {
			problems = append(problems, fmt.Sprintf(
				"%s serves %q to whoever requires it, and does not provide it", m.Module, to))
		}
	}
	for to, where := range m.Binds {
		if !strings.HasPrefix(where, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s binds %q at %q, which is not an absolute path", m.Module, to, where))
		}
		var wanted bool
		for _, w := range m.Wants() {
			if w == to {
				wanted = true
			}
		}
		if !wanted {
			// Being told about something you never asked for would write a file describing a
			// machine this one has no business talking to.
			problems = append(problems, fmt.Sprintf(
				"%s binds %q and does not require it", m.Module, to))
		}
	}
	if c := m.Certificate; c != nil {
		if !strings.HasPrefix(c.Into, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s wants its certificate at %q, which is not an absolute path", m.Module, c.Into))
		}
		if c.Authority != "" && !strings.HasPrefix(c.Authority, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s wants the authority at %q, which is not an absolute path",
				m.Module, c.Authority))
		}
	}
	for name, where := range m.Needs {
		if !strings.HasPrefix(where, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s needs %q at %q, which is not an absolute path", m.Module, name, where))
		}
		if name == "" {
			problems = append(problems, m.Module+" needs a secret with no name")
		}
	}
	for to, where := range m.Secrets {
		if !strings.HasPrefix(where, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s keeps the credential for %q at %q, which is not an absolute path",
				m.Module, to, where))
		}
		var wanted bool
		for _, w := range m.Wants() {
			if w == to {
				wanted = true
			}
		}
		if !wanted {
			problems = append(problems, fmt.Sprintf(
				"%s wants the credential for %q and does not require it", m.Module, to))
		}
	}
	for to, where := range m.Grants {
		if !strings.HasPrefix(where, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s grants %q into %q, which is not an absolute path", m.Module, to, where))
		}
		var offered bool
		for _, o := range m.Offers() {
			if o == to {
				offered = true
			}
		}
		if !offered {
			problems = append(problems, fmt.Sprintf(
				"%s grants %q to its consumers and does not provide it", m.Module, to))
		}
	}
	for to, where := range m.Receives {
		if !name.MatchString(to) {
			problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to))
		}
		if !strings.HasPrefix(where, "/") {
			problems = append(problems, fmt.Sprintf(
				"%s receives %q at %q, which is not an absolute path", m.Module, to, where))
		}
		var offered bool
		for _, o := range m.Offers() {
			if o == to {
				offered = true
			}
		}
		if !offered {
			// Receiving contributions to something you do not provide would create a file nobody
			// ever writes to, on a machine where nothing asked for it.
			problems = append(problems, fmt.Sprintf(
				"%s receives contributions to %q and does not provide it", m.Module, to))
		}
	}
	for i, r := range m.Resources {
		id, _ := r["id"].(string)
		if id == "" {
			problems = append(problems, fmt.Sprintf("resource %d has no id", i))
		}
		if _, ok := r["type"].(string); !ok {
			problems = append(problems, fmt.Sprintf("resource %q has no type", id))
		}
	}

	if len(problems) > 0 {
		sort.Strings(problems)
		return Manifest{}, fmt.Errorf("this manifest cannot be used:\n  - %s",
			strings.Join(problems, "\n  - "))
	}
	return m, nil
}

// Offers is everything this module can satisfy: its own name, and what it provides.
func (m Manifest) Offers() []string {
	out := []string{m.Module}
	for _, p := range m.Provides {
		out = append(out, p.Name)
	}
	sort.Strings(out)
	return out
}

// OffersAt is what this module provides at one scope, with its own name counted as node-scoped:
// a module is only ever itself on the machine it is installed on.
func (m Manifest) OffersAt(scope string) []string {
	var out []string
	if scope == ScopeNode {
		out = append(out, m.Module)
	}
	for _, p := range m.Provides {
		if p.At() == scope {
			out = append(out, p.Name)
		}
	}
	sort.Strings(out)
	return out
}