Files
mesh-controller/internal/catalogue/declaration.go
T
jschoubben 71f77617e3 Omit a consumer's identity where there is none
A contribution that is not a credential grant — a module offering
something to another on its own machine — has nobody to be identified
to, and was carrying an empty `as`. A field that is always present and
usually empty teaches a reader to ignore it, including when it is not.
2026-09-01 03:09:18 +02:00

641 lines
28 KiB
Go

package catalogue
// Turning a resolution into the declaration a node is sent.
//
// Separate from resolving because they answer different questions. Resolving asks *what should
// this machine run*; this asks *what does that look like as resources*, and the second is where
// settings are applied, generators are called, contributions are collected and credentials are
// placed. Both lived in one file until it was doing four jobs at once — which is the shape the
// system this replaces failed in, one import at a time.
import (
"encoding/json"
"fmt"
"sort"
"strings"
)
// SettingsBy is the layers that apply to each module, keyed by module name.
type SettingsBy map[string][]Layer
// Generator works out a module's resources for one node, where they cannot be written in advance.
type Generator interface {
// Resources for this node. Absent means the node is not part of whatever this generates,
// which is an ordinary answer rather than a failure — a machine assigned the module before it
// has an address on the network is in exactly that state.
Resources(node string) ([]map[string]any, bool, error)
}
// OpensPorts is a generator that also says what its resources accept connections on.
//
// **Separate from Generator, because most generators have nothing to say here** and requiring an
// empty method of each would be a cost paid everywhere for one caller.
//
// It exists because a static field cannot express this. A hub accepts connections from every node
// at other sites; a machine that is not a hub dials out and needs nothing open — and they are the
// same module. `listens` in a manifest is one answer for every machine that runs it, so the
// machine that most needs filtering, the one facing the public internet, was the one whose rule
// set would have closed its own overlay.
type OpensPorts interface {
// Listens is what this node accepts on because of what was computed for it. Nothing is the
// ordinary answer: most machines running a computed module open no port at all.
Listens(node string) ([]Listening, error)
}
// Grant is one consumer's credential, on the machine that must create it.
type Grant struct {
// Provision is what was required.
Provision string
// Consumer is the node that will use it.
Consumer string
// From is the module on that machine which asked.
//
// **Part of who this credential is for, not a label** (novox/hq 04-ISSUES/022). Together with
// Consumer it names one consumer; the node alone does not, because a machine routinely runs
// several modules wanting the same thing. It is also what the provider names the role or the
// bucket or the client after, so withdrawing one consumer does not take another's away.
From string
// Values are what that module contributed — the name it wants, and anything else the
// provision's own vocabulary defines.
Values map[string]any
// At is where the consuming machine is on the private network, empty if it is not on one.
//
// Passed in with the grant because it is a fact about another machine, and resolution answers
// questions about one. A provider that must reach back to its consumer — a reverse proxy is
// the whole reason this exists — otherwise has to know how the mesh names machines.
At string
// Sealed is the credential, closed to the providing node.
Sealed string
}
// Rendering is everything needed to turn a resolution into the declaration a node is sent.
type Rendering struct {
// Certificate is what the mesh issued for this machine's internal name, and the mesh's own
// certificate. Both public — the key they belong to never left the machine.
Certificate string
Authority string
// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
// name the module gave it.
Needed map[string]map[string]string
// Mesh is every node's address on the private network, which is what a rule saying "from the
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
// a fact about the mesh, and resolution answers questions about one machine.
Mesh []string
// Names is every machine's internal name and its address, for containers to be given.
//
// **A container does not inherit the machine's names**, so every internal name the mesh wrote
// is invisible to what the machine runs. Given here rather than looked up on the machine,
// because which machines exist is a fact about the mesh.
Names map[string]string
Settings SettingsBy
Generators map[string]Generator
// Grants are the credentials this node must create, for the provisions it offers. Passed in
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
// resolution answers questions about one machine.
Grants []Grant
}
// Declaration is everything the resolved modules put on the node, with settings applied.
//
// Resource identities are prefixed with the module they came from. Two modules may reasonably
// both call something "config", and without this the second would silently replace the first —
// the node applying one of them and reporting success.
func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
// Where each provision's credentials land, so a contribution can name the file rather than
// carry a value the mesh does not have.
directories := map[string]string{}
for _, m := range r.Modules {
for provision, where := range m.Grants {
directories[provision] = where
}
}
given, err := r.contributions(with.Settings, with.Grants, directories)
if err != nil {
return nil, err
}
// Once, from every module's listens -- not per module. A module receiving only its own ports
// would write a rule set that closed every other module on the machine.
rules, err := r.Filtering(with.Generators)
if err != nil {
return nil, err
}
filtering := AsNftables(rules, with.Mesh)
var out []map[string]any
for _, m := range r.Modules {
resources := m.Resources
// What the mesh computes for this module goes FIRST, before the module's own resources.
//
// **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so
// whatever the mesh writes down is the order a machine applies. A module's service or
// container routinely depends on one of these files; nothing here ever depends on a
// module's resources, because none of it is computed from them.
//
// Appended, this was wrong in a way that only showed on the first apply and then healed:
// the service started before its certificate or its rule set existed, failed, and the next
// reconcile fixed it. A fault that repairs itself on the second attempt is worse than one
// that does not, because what gets remembered is that it works.
var first []map[string]any
if f := m.Filtering; f != nil {
first = append(first, map[string]any{
"id": FilteringID(), "type": "file", "path": f.Into,
"content": filtering, "mode": "0600",
})
}
if c := m.Certificate; c != nil {
if with.Certificate == "" {
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
// with no certificate does not start, and the reason is somewhere else entirely.
return nil, fmt.Errorf(
"%s wants a certificate for this machine and none was issued", m.Module)
}
first = append(first, map[string]any{
"id": CertificateID(), "type": "file", "path": c.Into,
// Public. It travels in the open like any other file, because it is a statement
// about a key rather than the key.
"content": with.Certificate, "mode": "0644",
})
if c.Authority != "" {
first = append(first, map[string]any{
"id": AuthorityID(), "type": "file", "path": c.Authority,
"content": with.Authority, "mode": "0644",
})
}
}
for _, name := range sortedKeys(m.OwnSecrets) {
sealed := with.Needed[m.Module][name]
if sealed == "" {
// Declared and not made. Refused rather than skipped: a module whose own
// credential is silently absent starts, fails to authenticate, and the reason is
// three layers away from the machine reporting it.
return nil, fmt.Errorf(
"%s needs a secret called %q and none was made for it", m.Module, name)
}
first = append(first, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name], "sealed": sealed,
})
}
for _, to := range sortedKeys(m.Secrets) {
var found *Needed
for i, n := range r.Needs {
// **This module's need, not the provision's** (novox/hq 04-ISSUES/022). Matching
// on the name alone, every consumer of a provision took whichever credential
// happened to be last in the list — so on a node with two of them, one module
// would be given the other's password and fail to authenticate with a valid
// credential belonging to somebody else.
if n.Name == to && n.For == m.Module {
found = &r.Needs[i]
}
}
if found != nil && found.ByRecord && found.Sealed == "" {
// Answered by a record whose key has not been supplied since this consumer was
// put on it. **Refused, not skipped.** The mesh discarded the plaintext when the
// key was accepted and cannot seal another, so a machine that resolved cleanly
// would receive no file at all and fail at whatever tried to read it — which is
// the outcome ADR 0024 exists to avoid, arrived at politely.
return nil, fmt.Errorf(
"%s on this machine uses the licence %q and no key has been sealed to it. "+
"The mesh cannot make one; supply it again with `licence key %s`",
m.Module, found.From, found.From)
}
if found == nil || found.Sealed == "" {
// Answered on this machine, or answered by a node the mesh could not seal to.
// Nothing to write either way, and writing an empty credential file would be
// worse than none: something would read it and fail authenticating.
continue
}
first = append(first, map[string]any{
"id": SecretID(to), "type": "file", "path": m.Secrets[to],
"sealed": found.Sealed,
})
}
for _, to := range sortedKeys(m.Grants) {
for _, g := range with.Grants {
if g.Provision != to {
continue
}
if g.From == "" {
// Nothing on that machine asks for this any more. Skipped here rather than
// where grants are gathered, so the rule holds whoever gathers them.
//
// **This is how a credential is withdrawn.** The provisioner removes what
// nobody asks for, and it can only do that if the mesh stops asking — a
// consumer that was unassigned would otherwise keep a working login for ever,
// and nothing would say so.
continue
}
first = append(first, map[string]any{
"id": GrantID(to, g.Consumer+"."+g.From),
"type": "file",
"path": grantPath(m.Grants[to], g.Consumer, g.From),
"sealed": g.Sealed,
})
}
}
for _, to := range sortedKeys(m.Binds) {
var found *Needed
for i, n := range r.Needs {
if n.Name == to && n.For == m.Module {
found = &r.Needs[i]
}
}
if found == nil {
// Answered on this same machine rather than from elsewhere in the mesh.
//
// **Still written.** It used to be skipped, reasoning that "it is on this node"
// is a fact nobody needs — and that is true of the *location* and false of
// everything beside it. A binding also carries what the provider said a consumer
// must know, which is the port; a consumer cannot invent that, and got an absent
// file with no explanation. A build machine sharing a node with the registry it
// pushes to sat in a loop saying it could not read its own binding.
here := here(r, to)
if here == nil {
// Nothing in this node's set offers it either, so there is genuinely nothing
// to say. Resolution has already refused anything unanswerable, so this is a
// requirement met by the module itself.
continue
}
found = here
}
file, err := boundFile(*found, m.Binds[to], ConsumerIdentity(r.Node, m.Module))
if err != nil {
return nil, err
}
first = append(first, file)
}
for _, to := range sortedKeys(m.Receives) {
file, err := receivedFile(to, m.Receives[to], given[to])
if err != nil {
return nil, err
}
first = append(first, file)
}
if m.Computed != "" {
generator, known := with.Generators[m.Computed]
if !known {
return nil, fmt.Errorf(
"%s says its resources are computed by %q, and this control plane has no %q",
m.Module, m.Computed, m.Computed)
}
generated, part, err := generator.Resources(r.Node)
if err != nil {
return nil, err
}
if !part {
// Assigned, and not yet part of what this generates. Nothing to put on the
// machine, which is different from an error: a node given the network module
// before it has an address is in exactly that state, briefly.
continue
}
resources = generated
}
// Now, and not before: a module whose resources are computed replaces them wholesale, and
// merging earlier would throw away the files it still needs.
resources = append(append([]map[string]any{}, first...), resources...)
// Every container is given the mesh's names. Not a choice a module makes: a module that
// listed them would go stale the day a machine joins, and one that did not would be a
// module whose containers cannot reach anything by name.
//
// A container that was given names of its own keeps them and gets the mesh's beside them:
// the mesh does not know what else a workload needs to reach, and taking something away
// to add something is not what "also" means.
if len(with.Names) > 0 {
resources = withMeshNames(resources, with.Names)
}
// What this module may name from inside one of its own files. Gathered once per module
// rather than per file, because it is a fact about the module.
sealed, err := sealedFor(m, r.Needs, with)
if err != nil {
return nil, err
}
// And what its bindings say, for the half of a connection that is not secret.
known := knownFor(m, r.Needs, r.Node)
for _, unsettled := range resources {
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
if err != nil {
return nil, err
}
copied := map[string]any{}
for k, v := range resource {
copied[k] = v
}
// **After settings, and that is the whole reason it is here.** A module's file
// content is where a setting lands, so a placeholder may only exist once the setting
// has been put in — filling secrets first would look at content that is not yet what
// the machine receives.
if err := intoFile(copied, sealed, m.Module); err != nil {
return nil, err
}
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
// holds it in the clear — so there is nothing for the host to be the only witness of,
// and sending it already filled in means the host learns no new field.
if err := boundInto(copied, known, m.Module); err != nil {
return nil, err
}
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
// A service saying what it reflects names resources within its own module, so those
// are prefixed too or they would point at nothing.
//
// **Unless it already names one.** A module may reflect a file another module put on
// the machine — the case this exists for is a resolver restarting when the mesh
// rewrites the names, which are computed by the mesh and belong to its module, not to
// the daemon's. Written `<module>.<id>`, and a dot is what marks it as already
// answered: prefixing it again would point at nothing, silently, and the daemon would
// serve the old names for ever while everything reported success.
if reflects, ok := resource["restart-on"].([]any); ok {
var renamed []any
for _, id := range reflects {
named := fmt.Sprint(id)
if strings.Contains(named, ".") {
renamed = append(renamed, named)
continue
}
renamed = append(renamed, m.Module+"."+named)
}
copied["restart-on"] = renamed
}
out = append(out, copied)
}
}
return out, nil
}
// Contribution is one module telling the answer to a requirement what it needs from it.
type Contribution struct {
// From is the module that said it, so the provider and a person reading the file can tell
// which route belongs to what.
From string `json:"from"`
// Node is the machine it said it from, empty when that is this one.
//
// A provision answered from anywhere in the mesh has consumers on other machines, and the
// provider has to know who they are — a database told to create a password and not who for
// cannot do anything with it. Contributions were node-local until this, which meant the one
// case that most needed them was the one they did not reach.
Node string `json:"node,omitempty"`
// At is where that machine is on the private network, empty when it is not on one or when it
// is this machine.
//
// The mesh knows it and a provider should not have to derive it. A reverse proxy is told
// *send traffic to this consumer* and has to open a connection — so without this every
// provider that reaches back to a consumer would have to know how the mesh names machines,
// which is a convention leaking into every module that implements a provision.
At string `json:"at,omitempty"`
// As is who this consumer is: what the provider should call the login it creates.
//
// **Said by the mesh rather than invented by the provisioner** (novox/hq 04-ISSUES/023). It
// used to be neither — the provisioner made a name, and the consumer, which has to present it
// to authenticate, had no way to learn it. One derivation reaches both ends, so they agree by
// construction.
// Omitted when there is none. A contribution that is not a credential grant — a module
// offering something to another on its own machine — has nobody to be identified to, and a
// field that is always present and usually empty teaches a reader to ignore it.
As string `json:"as,omitempty"`
// Secret is the file on this machine holding that consumer's credential, sealed to it.
//
// Named rather than carried, for the same reason the private network's key is: the mesh
// discarded the value and could not put it here if it wanted to. What is here is where to
// find it.
Secret string `json:"secret,omitempty"`
// Values are the module's own, with settings applied. What the keys mean is agreed by the
// requirement's name — everything providing `reverse-proxy` understands the same shape, which
// is what makes swapping one for another cost nothing.
Values map[string]any `json:"values"`
}
// grantPath is where one consumer's sealed credential lands on the providing machine.
//
// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a
// node named like something else in that directory cannot collide with it.
// One file per consumer, and a consumer is a module on a machine (novox/hq 04-ISSUES/022).
//
// Named after both. Named after the machine alone, two modules on one node wrote to one path: the
// second overwrote the first, and the provisioner — reading a directory — saw one consumer where
// there were two.
func grantPath(directory, consumer, module string) string {
return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret"
}
// contributions collects what every module in this set contributes, by requirement.
//
// Ordered by contributing module, because the result becomes a file on a machine and a file whose
// lines move about is a file that looks changed when nothing changed.
func (r Resolution) contributions(settings SettingsBy, grants []Grant,
directories map[string]string) (map[string][]Contribution, error) {
out := map[string][]Contribution{}
modules := append([]Manifest{}, r.Modules...)
sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module })
// What consumers on other machines asked for. Merged in with this machine's own, because from
// the provider's side they are the same thing — somebody wanting something — and a provider
// that had to read two lists would be a provider that reads one of them.
sorted := append([]Grant{}, grants...)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Provision != sorted[j].Provision {
return sorted[i].Provision < sorted[j].Provision
}
if sorted[i].Consumer != sorted[j].Consumer {
return sorted[i].Consumer < sorted[j].Consumer
}
return sorted[i].From < sorted[j].From
})
for _, g := range sorted {
if g.From == "" {
// As above: nothing on that machine asks for this any more, so the provider is not
// told about it and withdraws the login on its next pass.
continue
}
out[g.Provision] = append(out[g.Provision], Contribution{
From: g.From, Node: g.Consumer, At: g.At, Values: g.Values,
As: ConsumerIdentity(g.Consumer, g.From),
Secret: grantPath(directories[g.Provision], g.Consumer, g.From),
})
}
for _, m := range modules {
for _, to := range sortedKeys(m.Contributes) {
// Settings reach a contribution the same way they reach a file. A route's hostname is
// exactly the kind of thing that differs between one mesh and the next, and a module
// that could not have it set would have to be edited to be reused.
values, err := settle(m.Contributes[to], settings[m.Module], nil,
m.Module+" contributing to "+to)
if err != nil {
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
}
return out, nil
}
// receivedFile is the file a provider is given its consumers' contributions in.
func receivedFile(requirement, path string, given []Contribution) (map[string]any, error) {
if given == nil {
// Nobody contributed. The file is still written, empty, rather than left absent: a
// provider that finds no file cannot tell "nothing asked for me" from "the mesh never
// wrote it", and the two want completely different responses.
given = []Contribution{}
}
// The note goes *inside* the document, not above it. The first version wrote a `//` header
// and produced a file that says "do not edit" to a person and fails to parse for the program
// meant to read it — which is the whole audience.
body, err := json.MarshalIndent(map[string]any{
"contributions": 1,
"requirement": requirement,
"generated": "by the mesh — do not edit; replaced whenever a module contributing to " +
requirement + " arrives or leaves",
"given": given,
}, "", " ")
if err != nil {
return nil, err
}
return map[string]any{
"id": ReceivedID(requirement), "type": "file", "path": path, "mode": "0644",
"content": string(body) + "\n",
}, nil
}
// sortedKeys is map iteration made repeatable, which everything written to a machine needs.
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// boundFile is what a module is told about something it requires from another machine.
//
// Where it is and what the providing module said about using it. **No credential**, and the file
// says so rather than leaving a reader to wonder whether one was meant to be there — a missing
// field looks like a bug, and a stated absence looks like a boundary.
func boundFile(n Needed, path, as string) (map[string]any, error) {
// A record has no machine and no address. Saying so is the difference between a reader
// concluding "somewhere with no address" and concluding the mesh failed to fill something in.
where := any(n.At)
if n.ByRecord {
where = "a record in this mesh, not a machine"
}
body, err := json.MarshalIndent(map[string]any{
"binding": 1,
"provision": n.Name,
"from": n.From,
"at": where,
// Who this module is at the other end. **The half that was missing** (novox/hq
// 04-ISSUES/023): a consumer was told the address, the port and where its password is,
// and not the name it must present — which the provisioner had invented.
"as": as,
"serves": n.Serves,
// **Where the credential is, not what it is.** It stopped being true that the mesh
// cannot issue one when 021 was fixed, and a comment asserting a fact about the mesh that
// has become false is worse than none — somebody reads it and stops looking.
"generated": "by the mesh — do not edit; replaced whenever this changes. " +
"The credential is not here: it is sealed, in the file this module's manifest " +
"names under `secrets`",
}, "", " ")
if err != nil {
return nil, err
}
return map[string]any{
"id": BoundID(n.Name), "type": "file", "path": path, "mode": "0644",
"content": string(body) + "\n",
}, nil
}
// ContributionsFrom is what one module on this node asked of one requirement, settled.
//
// Exported because a provider's grants are assembled from its consumers' resolutions, one machine
// at a time, and the alternative was for the control plane to reimplement settling.
//
// **One module, not one machine** (novox/hq 04-ISSUES/022). This used to take a requirement alone
// and refuse whenever two modules wanted it — correctly, given what it had: the credential was
// keyed by node, so the two would have shared one, and sharing is worse than refusing. But the
// arrangement refused is the ordinary one. A node running eight services against one database is
// not an edge case; it is what a machine looks like. Now each consumer has its own credential and
// there is nothing left to refuse.
func (r Resolution) ContributionsFrom(requirement, module string, settings SettingsBy) (
map[string]any, bool, error) {
all, err := r.contributions(settings, nil, nil)
if err != nil {
return nil, false, err
}
for _, g := range all[requirement] {
if g.From == module {
return g.Values, true, nil
}
}
// Nothing on that machine asks for this any more. Said as "not found" rather than as an
// error: it is how a credential is withdrawn, and the provider removes what nobody asks for.
return nil, false, nil
}
// here is the module on this same machine that answers a requirement, as a binding.
//
// **Only when the provider said something a consumer must know.** That is the line: the original
// reasoning — a file saying "it is on this node" is a fact nobody needs — is right about the
// location and wrong about everything beside it. A shell is answered here and there is nothing to
// say about it. A registry is answered here and the consumer still cannot guess the port.
//
// The address is this machine's own name on the private network when it has one, and loopback
// when it does not — a machine not on the network still reaches itself, and naming it by a name
// nothing resolves would be worse than naming it by an address that always works.
func here(r Resolution, requirement string) *Needed {
for _, m := range r.Modules {
serves, said := m.Serves[requirement]
if !said || len(serves) == 0 {
continue
}
at := r.At
if at == "" {
at = "127.0.0.1"
}
return &Needed{Name: requirement, From: r.Node, At: at, Serves: serves}
}
return nil
}
// withMeshNames gives every container in a set the mesh's names.
//
// Copied rather than edited in place: these maps come from a module's manifest, and mutating one
// would change what the catalogue holds for every other machine running that module.
func withMeshNames(resources []map[string]any, names map[string]string) []map[string]any {
out := make([]map[string]any, 0, len(resources))
for _, r := range resources {
if r["type"] != "container" {
out = append(out, r)
continue
}
// A container on the machine's own network shares its hosts file already, and a runtime
// refuses to write one for it. Adding names there would be an argument the runtime
// rejects, which fails the whole container for something it did not need.
if network, on := r["network"].(string); on && network == "host" {
out = append(out, r)
continue
}
copied := map[string]any{}
for k, v := range r {
copied[k] = v
}
var given []any
if already, ok := copied["hosts"].([]any); ok {
given = append(given, already...)
}
for _, name := range sortedKeys(names) {
given = append(given, name+":"+names[name])
}
copied["hosts"] = given
out = append(out, copied)
}
return out
}