novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity
Files
db7cd273f0304f1f633e9f6804d6f44387f62983
mesh-host/internal/compose/resolution/declaration.go
T

2523 lines
111 KiB
Go
Raw Blame History

package resolution
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
)
// rootInto composes a user's `root` for the node-engine: taken away when it asserts nothing (empty — a
// machine where agents run as the operator) or when the engine is older than the field and parses
// strictly; kept as "never" otherwise.
func rootInto(resource map[string]any, with mcat.Rendering) {
if resource["type"] != "user" {
return
}
value, has := resource[mcat.RootField]
if !has {
return
}
if s, _ := value.(string); s == "" || !with.JudgesRoot {
delete(resource, mcat.RootField)
}
}
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
// not been asked. Unassigned is not an error here: a module with no `listens` never needed one,
// and a caller composing a declaration without a store still gets something coherent.
func renderingMachinePort(r mcat.Rendering, module string, wanted int) int {
if at, given := r.Given[module][wanted]; given {
return at
}
if at, known := r.Ports[module][wanted]; known {
return at
}
return wanted
}
// Declaration is everything the resolved modules put on the node, with settings applied.
//
// Resource identities are prefixed with the module they came from. Two modules may reasonably
// both call something "config", and without this the second would silently replace the first —
// the node applying one of them and reporting success.
func Declaration(r mcat.Resolution, with mcat.Rendering) ([]map[string]any, error) {
composed, err := Compose(r, with)
if err != nil {
return nil, err
}
return composed.Resources, nil
}
// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275):
// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that
// fails for any other reason without reading the words.
type NotMadeError struct {
Module, Name string
}
func (e *NotMadeError) Error() string {
return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name)
}
// LeftOut is which of this machine's modules a declaration composed with these settings leaves
// out, and why (novox/hq ADR 0163, rule 6): each whose stored settings its definition can no longer
// compose. Empty when every module composes. The same judgement SetSettings makes before storing.
func LeftOut(r mcat.Resolution, settings mcat.SettingsBy, adopted bool) map[string]string {
out := map[string]string{}
for module, why := range LeftOutBecause(r, settings, adopted) {
out[module] = why.Error()
}
return out
}
// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody
// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue
// 380). An UnreadManifestError for a stored manifest this controller cannot read whole.
func LeftOutBecause(r mcat.Resolution, settings mcat.SettingsBy, adopted bool) map[string]error {
out := map[string]error{}
for _, m := range r.Modules {
if why := UnknownFieldReason(m); why != "" {
out[m.Module] = &UnreadManifestError{Module: m.Module, said: why}
continue
}
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
out[m.Module] = err
}
}
return out
}
// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know
// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine.
type UnreadManifestError struct {
Module string
said string
}
func (e *UnreadManifestError) Error() string { return e.said }
// Compose is Declaration with the owner of every resource said.
func Compose(r mcat.Resolution, with mcat.Rendering) (mcat.Composed, error) {
owner := map[string]string{}
received := map[string]map[string][]mcat.Contribution{}
leftOut := map[string]string{}
var foreseen, unplaced, handedOnly []string
handed := &handedFiles{plans: map[string]mcat.Plan{}}
resources, err := resolutionCompose(r, with, owner, received, leftOut, &foreseen, &unplaced, &handedOnly, handed)
if err != nil {
return mcat.Composed{}, err
}
sort.Strings(foreseen)
if with.BusMembership != "" {
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
// secret and placed where the host looks for exactly this (design 28, task 5.2).
resources = append(resources, map[string]any{
"id": BusMembershipID(), "type": "file", "path": mcat.BusMembershipPath,
"sealed": with.BusMembership, "mode": "0600",
})
}
sort.Strings(unplaced)
sort.Strings(handedOnly)
out := mcat.Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut,
Foreseen: foreseen, Unplaced: unplaced, HandedOnly: handedOnly, Filled: handed.paths}
if plans := handed.finished(r.Needs); len(plans) > 0 {
out.Plans = plans
}
return out, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
// BusMembershipPath is where the host reads it — the same constant on both sides.
func BusMembershipID() string { return "bus-membership" }
func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string]string,
received map[string]map[string][]mcat.Contribution, leftOut map[string]string,
foreseen, unplaced, handedOnly *[]string, handed *handedFiles) ([]map[string]any, error) {
// **A setting is judged where it is stored, and an impossible one costs a module, not a
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
// this module uncomposable; it is left out of the declaration — its held things kept, its
// containers untouched, the machine told so by name — and the machine is told everything else.
// Before placing, because a placement is a setting too.
left := LeftOut(r, with.Settings, with.Adopted)
kept := make([]mcat.Manifest, 0, len(r.Modules))
var stillBackedUp []mcat.Manifest
for module, why := range with.NotPlaced {
if _, already := left[module]; !already {
left[module] = why
}
}
for _, m := range r.Modules {
if why, isLeft := left[m.Module]; isLeft {
if leftOut != nil {
leftOut[m.Module] = why
}
// **Its data is still copied** (novox/hq ADR 0262): a module left out runs nothing new, and
// the data it already holds on the machine is the reason to keep copying it. Only its data,
// as the backup holder's lines are derived from it, and the directories they name.
stillBackedUp = append(stillBackedUp, backupView(m))
continue
}
kept = append(kept, m)
}
r.Modules = kept
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every
// reader below — the binding files, the sealed secrets, the grant paths a contribution
// names — sees a concrete place and none learns the vocabulary.
// Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a
// slice element written in place would carry the first node's places into the second's.
placed := make([]mcat.Manifest, len(r.Modules))
for i, m := range r.Modules {
var err error
if placed[i], err = placedManifest(m, with); err != nil {
return nil, err
}
}
r.Modules = placed
// Where each provision's credentials land, so a contribution can name the file rather than
// carry a value the mesh does not have.
directories := map[string]string{}
for _, m := range r.Modules {
for provision, where := range m.Grants {
directories[provision] = where
}
}
given, err := resolutionContributions(r, with.Settings, with.Grants, directories)
if err != nil {
return nil, err
}
// A workload on THIS machine contributes the port it declared, and the machine may have
// published it somewhere else (novox/hq ADR 0066). The mirror of the redirect below: 038 fixed
// what a consumer is TOLD about a provider, and this is what a workload TELLS the provider about
// itself — gitea declaring 3000, published as 20000:3000, and the co-located proxy dialling 3000,
// where nothing listens, for every request.
//
// **The contributing module's assignment, never the provider's.** The port belongs to the
// workload; using the map of whatever answers the requirement would move a route to wherever the
// proxy happens to be published. A contribution carried here from another machine is left exactly
// as it is — its port is that machine's to assign, and this map knows nothing about it.
for provision := range given {
for i := range given[provision] {
said := &given[provision][i]
if said.Node != "" && said.Node != r.Node {
continue
}
said.Values = atMachinePort(said.Values, said.From, with.Ports)
}
}
// A provider answered on this same machine never passed through the walk that works out what a
// provider on ANOTHER machine serves (novox/hq 04-ISSUES/038, ADR 0066). That walk does two
// things — it derives the served facts with the provider node's port assignments, and it settles
// them with that node's settings layers — and the same-node paths (resolve.go's servedHere, and
// here() below) did neither, because both run while resolving, before either is known.
//
// The port was the first half to be noticed (038). The second half is worse and quieter: a served
// value the OPERATOR supplied — a certificate authority's root, which is the one kind of value a
// manifest cannot carry, because it differs on every mesh — arrived as the manifest's empty
// default. A proxy then wrote an empty CA bundle, fell back to the system trust store, could not
// verify the internal authority, and stopped issuing with nothing saying why.
//
// So it is re-derived here, in full, exactly as the cross-node path derives it. **After the
// closure is known**, which also removes an order dependence: the resolver built these needs
// mid-walk from whichever modules had been chosen by then, so what a co-located binding carried
// depended on the order somebody happened to assign things in.
for i := range r.Needs {
if r.Needs[i].ByRecord || r.Needs[i].From != r.Node {
continue
}
serves, answered, err := resolutionServedOnThisMachine(r, r.Needs[i].Name, with)
if err != nil {
return nil, err
}
if answered {
r.Needs[i].Serves = serves
}
}
// Once, from every module's listens -- not per module. A module receiving only its own ports
// would write a rule set that closed every other module on the machine. Each module's per-node
// exposure settings override its listens' source first (novox/hq ADR 0046).
rules, err := Rules(r, with)
if err != nil {
return nil, err
}
// **A machine that has not said which links face outside is sent no filter** (novox/hq ADR
// 0140). The whole chain is written around those links: with none, the rule that lets this
// machine's own guests keep working would name an empty set, which nftables refuses, and a rule
// set that does not load is a machine filtering nothing while its unit reports success. Refused
// here, where a person reads it, rather than on the machine — and the machine keeps the filter
// it already has.
if filters := resolutionFiltersHere(r); filters != "" && len(with.OutwardLinks) == 0 {
return nil, fmt.Errorf(
"%s cannot be sent a filter: it has not reported which of its links face outside, and "+
"every rule in the chain is written around them. It reports that on each apply; "+
"`node show %s` says whether it has. Until then %s is not sent, and the machine "+
"keeps the filter it has", r.Node, r.Node, filters)
}
filtering := AsNftablesWithLocal(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
with.OutwardLinks, with.TunnelInterface, with.LocalNetworks)
// **A variable two modules set is refused whether or not anything places it** (novox/hq ADR
// 0203 §5): the account has one environment, and a machine whose holder arrives later should not
// be the moment two modules are found to disagree about it.
if err := variablesSetOnce(r.Modules); err != nil {
return nil, err
}
generated, err := resolutionGeneratedHere(r, with)
if err != nil {
return nil, err
}
var out []map[string]any
for _, m := range r.Modules {
if with.Adopted && m.Filtering != nil {
// Nothing of a module that loads a filter, on an adopted node: its table would drop
// by default and hold accepts, and the found firewall stays in force. Every resource,
// not only the rule set — its service must not run, and a node returned to adopted
// stops it by the ordinary removal of what is no longer declared.
continue
}
resources := m.Resources
// What the mesh computes for this module goes FIRST, before the module's own resources.
//
// **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so
// whatever the mesh writes down is the order a machine applies. A module's service or
// container routinely depends on one of these files; nothing here ever depends on a
// module's resources, because none of it is computed from them.
//
// Appended, this was wrong in a way that only showed on the first apply and then healed:
// the service started before its certificate or its rule set existed, failed, and the next
// reconcile fixed it. A fault that repairs itself on the second attempt is worse than one
// that does not, because what gets remembered is that it works.
var first []map[string]any
if f := m.Filtering; f != nil {
first = append(first, map[string]any{
"id": mcat.FilteringID(), "type": "file", "path": f.Into,
"content": filtering, "mode": "0600",
})
}
// The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written
// where the intrusion-prevention holder owns them. Like the rule set above: gathered from all
// modules, written by the one that holds the role.
if j := m.Jailing; j != nil {
first = append(first, jailsInto(r.Modules, j)...)
}
if c := m.Certificate; c != nil {
if with.Certificate == "" {
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
// with no certificate does not start, and the reason is somewhere else entirely.
return nil, fmt.Errorf(
"%s wants a certificate for this machine and none was issued", m.Module)
}
first = append(first, map[string]any{
"id": mcat.CertificateID(), "type": "file", "path": c.Into,
// Public. It travels in the open like any other file, because it is a statement
// about a key rather than the key.
"content": with.Certificate, "mode": "0644",
})
if c.Authority != "" {
first = append(first, map[string]any{
"id": mcat.AuthorityID(), "type": "file", "path": c.Authority,
"content": with.Authority, "mode": "0644",
})
}
}
if m.BusUsers != "" {
// **The claim authorises it, not the field.** This file holds every user's password
// hash, so a module that could ask for it could read every credential on the bus.
// Checked from this manifest alone, which is the cheapest check there is: whether some
// other module also claims the seat is resolution's business elsewhere, and one holder
// mesh-wide is already guaranteed.
if !m.ClaimsSeat("mesh-broker") {
return nil, fmt.Errorf(
"%s asks for the mesh's user list and does not claim mesh-broker. That file "+
"holds every user's password hash, so the seat is what authorises it",
m.Module)
}
if with.BusUsers == "" {
// Asked for and not composed. Refused rather than skipped, for the reason a
// certificate is: a bus with no user list refuses every connection in the mesh, and
// an empty file would look like a configuration problem on the machine.
return nil, fmt.Errorf(
"%s holds mesh-broker and the mesh composed no user list, so the bus would "+
"refuse every connection", m.Module)
}
first = append(first, map[string]any{
"id": mcat.BusUsersID(), "type": "file", "path": m.BusUsers,
"content": with.BusUsers,
// Readable by the server and nothing else. Hashes rather than passwords, so this is
// not a set of working credentials — but a list of every user in the mesh is worth
// keeping to the one process that needs it.
"mode": "0600",
})
}
// **A secret family's members, as given** (novox/hq ADR 0283): one file each at the family's path, and
// nothing for a member not given — the mesh never makes one, and the module says it waits for it.
for _, name := range sortedKeys(with.Needed[m.Module]) {
own, family, ok := m.OwnSecrets.Lookup(name)
if !ok || family == "" || with.Needed[m.Module][name] == "" {
continue
}
// A per-secret owner (novox/hq ADR 0436 §1) stands in for the module's secrets-owner, when
// the manifest named one for this secret (or the family it belongs to).
owner := m.SecretsOwner
if own.Owner != "" {
owner = own.Owner
}
first = append(first, ownedBy(owner, map[string]any{
"id": mcat.NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name],
}))
}
// **A handed module's own secrets are the vault's** (design 57 V4, issue 647): each file holding one is
// declared placed and without a value, filled by the machine's provisioner client from the vault; the
// controller makes and seals none of them here.
handsSecrets := Handing(with, m)
for _, name := range sortedKeys(m.OwnSecrets.Plain()) {
if handsSecrets {
owner := resolutionOwnSecretOwner(r, m, name)
first = append(first, ownedBy(owner, map[string]any{
"id": mcat.NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, mcat.HandedSecretField: name,
}))
continue
}
// **A core credential is the vault's box** (ADR 0349 §16, issue 755): carried as the vault sealed it to this
// machine, which the controller cannot open; the controller's own sealed value is not sent with it.
if box, ok := renderingVaultSealedOf(with, m.Module, name); ok {
if string(box) == string(mcat.ForeseenVaultBox) {
*foreseen = append(*foreseen, m.Module+"/"+name)
}
owner := resolutionOwnSecretOwner(r, m, name)
first = append(first, ownedBy(owner, map[string]any{
"id": mcat.NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, mcat.VaultSealedField: box,
}))
continue
}
sealed := with.Needed[m.Module][name]
if sealed == "" && with.Foreseen[m.Module][name] {
// Not made, and the next send makes it: composed with a stand-in so that whatever
// else this composition would refuse is still found (novox/hq issue 275).
sealed = mcat.ForeseenSealed
*foreseen = append(*foreseen, m.Module+"/"+name)
}
if sealed == "" && m.OwnSecrets[name].IssuedBy == mcat.IssuedOutside {
// **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a
// family's member (ADR 0283), so the module reads its absence as not given.
continue
}
if sealed == "" {
// Declared and not made. Refused rather than skipped: a module whose own
// credential is silently absent starts, fails to authenticate, and the reason is
// three layers away from the machine reporting it.
return nil, &NotMadeError{Module: m.Module, Name: name}
}
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
// root-owned 0600 file is one that process cannot read. Composed here rather than said in
// the manifest, because a manifest cannot say ${machine:account} safely — a node with no
// account has nothing to resolve it to, and then the runtime runs as root and the file
// stays root's.
first = append(first, ownedBy(resolutionOwnSecretOwner(r, m, name), map[string]any{
"id": mcat.NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
}))
}
// This module's tools bundles, where the machine runs the node's tool runtime (novox/hq
// ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's
// own resources for the same reason: the runtime's process names the files inside these
// and is restarted when one changes, so they are on the machine before it is.
if resolutionRuntimeHere(r) {
first = append(first, bundleArchives(m)...)
}
// Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before
// the module's own resources, and so before the container that mounts them: the host must
// find each present — refusing clearly if the operator has not provided it — before it
// starts anything that depends on it. The mesh creates, chowns and reconciles none of it;
// an `access` resource says only *this path must exist, and this module reaches it*.
// Where each is on THIS machine is the assignment's (novox/hq issue 153): placed by id
// where the operator said, the definition's default otherwise, refused where neither.
accesses, accessPaths, err := accessesFor(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
for _, a := range accesses {
first = append(first, map[string]any{
"id": mcat.AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.Mode,
})
}
for _, to := range m.SecretRequirements() {
for _, file := range m.SecretFiles(to) {
var found *mcat.Needed
for i, n := range r.Needs {
// **This module's need, not the provision's** (novox/hq 04-ISSUES/022). Matching
// on the name alone, every consumer of a provision took whichever credential
// happened to be last in the list — so on a node with two of them, one module
// would be given the other's password and fail to authenticate with a valid
// credential belonging to somebody else. And this file's local name, where the
// module keeps several (ADR 0094).
if n.Name == to && n.For == m.Module && n.Local == file.Local {
found = &r.Needs[i]
}
}
// **A credential the vault keeps for this holder alone** (issue 757): where the module is handed, its
// file is declared placed and without a value, and the machine's client fills it from the vault.
if found != nil && found.IssuedToHolder && Handing(with, m) {
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
"id": mcat.SecretID(mcat.SecretLocal(to, file.Local)), "type": "file", "path": file.Path, mcat.HandedProvisionField: mcat.PlanProvision{Name: mcat.SecretLocal(to, file.Local), Provision: to,
Local: file.Local, Provider: found.From, ProviderModule: found.Module},
}))
continue
}
if found != nil && found.ByRecord && found.Sealed == "" && !found.Manager {
// Answered by a record whose key has not been supplied since this consumer was
// put on it. **Refused, not skipped.** The mesh discarded the plaintext when the
// key was accepted and cannot seal another, so a machine that resolved cleanly
// would receive no file at all and fail at whatever tried to read it — which is
// the outcome ADR 0024 exists to avoid, arrived at politely.
//
// The manager holder is the one exception (novox/hq ADR 0050): an empty refresh token
// is a licence whose manager has not adopted one yet, a real waiting state rather than
// a lost key. It falls through to the skip below — its bound facts (carrying the
// manager's public key) are still delivered, which is what adoption needs to seal the
// first refresh token.
return nil, fmt.Errorf(
"%s on this machine uses the licence %q and no key has been sealed to it. "+
"The mesh cannot make one; supply it again with `licence key %s`",
m.Module, found.From, found.From)
}
if found == nil || found.Sealed == "" {
// Answered on this machine, or answered by a node the mesh could not seal to.
// Nothing to write either way, and writing an empty credential file would be
// worse than none: something would read it and fail authenticating.
continue
}
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
"id": mcat.SecretID(mcat.SecretLocal(to, file.Local)), "type": "file", "path": file.Path,
"sealed": found.Sealed,
}))
}
}
for _, to := range sortedKeys(m.Grants) {
for _, g := range with.Grants {
if g.Provision != to {
continue
}
if g.From == "" {
// Nothing on that machine asks for this any more. Skipped here rather than
// where grants are gathered, so the rule holds whoever gathers them.
//
// **This is how a credential is withdrawn.** The provisioner removes what
// nobody asks for, and it can only do that if the mesh stops asking — a
// consumer that was unassigned would otherwise keep a working login for ever,
// and nothing would say so.
continue
}
if g.Sealed == "" {
// Its offer presents no credential (form none, issue 1126): nothing to write, and an empty credential
// file would be read as one that failed. Who asked still reaches the provider in its contributions.
continue
}
first = append(first, ownedBy(resolutionProvisionsAs(r, m), map[string]any{
// One file per holder — the consumer's module with its local name after it
// where it keeps several (ADR 0094); the lab found two files with one id.
"id": mcat.GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)),
"type": "file",
"path": grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)),
"sealed": g.Sealed,
}))
}
}
for _, to := range sortedKeys(m.Binds) {
var found *mcat.Needed
for i, n := range r.Needs {
if n.Name == to && n.For == m.Module {
found = &r.Needs[i]
}
}
if found == nil {
// Answered on this same machine rather than from elsewhere in the mesh.
//
// **Still written.** It used to be skipped, reasoning that "it is on this node"
// is a fact nobody needs — and that is true of the *location* and false of
// everything beside it. A binding also carries what the provider said a consumer
// must know, which is the port; a consumer cannot invent that, and got an absent
// file with no explanation. A build machine sharing a node with the registry it
// pushes to sat in a loop saying it could not read its own binding.
here, err := here(r, to, with)
if err != nil {
return nil, err
}
if here == nil {
// Nothing in this node's set offers it either, so there is genuinely nothing
// to say. Resolution has already refused anything unanswerable, so this is a
// requirement met by the module itself.
continue
}
found = here
}
own, err := resolutionOwnNames(r, m, to, with.Settings[m.Module])
if err != nil {
return nil, err
}
as := mcat.ConsumerIdentity(r.Node, mcat.IdentitySource(m.Slug, m.Module))
// What the provider derives for THIS consumer, filled here where the consumer is
// known (novox/hq ADR 0202). The same fill knownFor does below, so the binding file
// and the module's `${bound:…}` substitutions cannot say different things.
told := *found
told.Serves, err = ServedTo(told.Serves, as)
if err != nil {
return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err)
}
file, err := boundFile(told, m.Binds[to], as, own)
if err != nil {
return nil, err
}
first = append(first, file)
}
for _, to := range sortedKeys(m.Receives) {
file, err := receivedFile(to, m.Receives[to], given[to])
if err != nil {
return nil, err
}
first = append(first, file)
if received[m.Module] == nil {
received[m.Module] = map[string][]mcat.Contribution{}
}
// Empty rather than absent when nobody contributed, for the reason the file is
// written empty: "nothing asked" and "never told" want different responses.
received[m.Module][to] = append([]mcat.Contribution{}, given[to]...)
}
if m.Keeps != "" && with.Kept != nil {
file, err := keptFile(m.Keeps, with.Kept)
if err != nil {
return nil, err
}
// Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723).
first = append(first, ownedBy(m.SecretsOwner, file))
} else if runs, ok := with.KeptRuns[m.Module]; m.Keeps != "" && ok {
file := make(map[string]any, len(runs))
for k, v := range runs {
file[k] = v
}
file["id"] = KeptID()
delete(file, "module")
first = append(first, file)
}
if m.Computed != "" {
mine, part := generated[m.Module]
if !part {
// Assigned, and not yet part of what this generates. Nothing to put on the
// machine, which is different from an error: a node given the network module
// before it has an address is in exactly that state, briefly.
continue
}
resources = mine
}
// Now, and not before: a module whose resources are computed replaces them wholesale, and
// merging earlier would throw away the files it still needs.
//
// **Except the module's own accounts, which go before even those** (novox/hq issue 213). What
// the mesh computes may belong to one: a module whose code runs as an account it declares has
// its secrets written owned by that account, and a file given to a user the machine does not
// have yet fails — so on the first apply the secrets were refused, the process started without
// them, and the second apply healed it, which is the fault the paragraph above describes.
// An account depends on nothing the mesh computes.
accounts, rest := accountsFirst(resources)
resources = append(append(accounts, first...), rest...)
// **A file delivered only while its optional requirement is bound is left out while it is
// absent** (novox/hq ADR 0330 §5), and the field is dropped from the rest: the node-engine
// parses strictly and knows no such field.
absent := AbsentFor(r, m.Module)
resources = withoutAbsent(resources, absent)
// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
// container got the whole roster as `--add-host` entries at creation, and a name that
// moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once
// the roster was made part of a container's identity so that could be caught, one name
// moving anywhere replaced every container in the mesh (issue 151). A container resolves a
// mesh name through its machine's resolver at the moment it asks, which the runtime is
// told once per machine, as a file, by the resolver's own module. The names a module
// declares for itself are its own and stay exactly as written: they are part of what the
// module is, and the mesh does not know what they mean.
// What this module may name from inside one of its own files. Gathered once per module
// rather than per file, because it is a fact about the module.
sealed, err := sealedFor(m, r.Needs, with)
if err != nil {
return nil, err
}
// And what its bindings say, for the half of a connection that is not secret.
known, err := knownFor(m, r.Needs, r.Node)
if err != nil {
return nil, err
}
// A requirement answered on this same machine is not in r.Needs — its binding file is
// written from `here` (above) — and so `${bound:…}` could not name it, though the file
// beside it said the same facts. Filled from the same answer, so the two cannot disagree.
for _, want := range m.Wants() {
if _, has := known[want]; has {
continue
}
answered, err := here(r, want, with)
if err != nil {
return nil, err
}
if answered == nil {
continue
}
local := *answered
local.For = m.Module
here, err := knownFor(m, []mcat.Needed{local}, r.Node)
if err != nil {
return nil, err
}
for provision, values := range here {
known[provision] = values
}
}
// And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is
// what nearly every consumer wants; the one that cannot use it is a machine's resolver
// configuration, which must reach the resolver before it can resolve anything — the resolver's
// own name included. Absent when the machine has no address yet, so a file naming it is refused
// rather than written with a blank where an address belongs.
for _, values := range known {
if address := with.Machines[values["at"]]; address != "" {
values["address"] = address
}
}
// And what the module is called through each requirement it contributes to (novox/hq
// 04-ISSUES/122) — the same composition its binding file carries.
for provision, values := range known {
own, err := resolutionOwnNames(r, m, provision, with.Settings[m.Module])
if err != nil {
return nil, err
}
withOwnNames(values, own)
}
// And where this node places the directories the module declared without a path
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
// — and, on an adopted machine, where the assignment says they already are, with the
// owner the data already has (novox/hq issue 153). Malformed placements are refused here.
placed, err := mcat.Places(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
dirs := dirsFor(m, with)
// And the machine underneath, which no binding of its own can tell it.
thisMachine := machineFacts(r, with.Names, with.MeshRange)
// **A definition that already holds the answer transcribed it** (novox/hq ADR 0202).
// Judged over what the module itself declares, and before anything is substituted: the
// mesh's own generated files — the binding, the contributions — legitimately carry the
// derived value, and after substitution so does every consumer's file, so this is the one
// moment the two can be told apart.
for _, own := range m.Resources {
if err := notTranscribed(own, known, m.Module); err != nil {
return nil, err
}
}
// **And every placeholder no pass fills where it stands is refused** (novox/hq issue 231):
// judged here, over the definition as written and before any pass, by the function the
// catalogue check runs — so a manifest registered by an older binary is judged too, and
// what a setting or a binding later puts into a file is its software's text, never swept.
for _, own := range m.Resources {
if problems := unconsumedPlaceholders(m.Module, own); len(problems) > 0 {
return nil, fmt.Errorf("%s", problems[0])
}
}
// Which of this module's files carry a secret, for the rule that a container may not read
// one of them as its environment without saying so (ADR 0086, issue 041).
secretFiles := secretFilesOf(resources)
// Which of this module's resources its preparation runs before, if it prepares anything.
prepareBefore := preparationTarget(m)
// Which found networks this machine's setting keeps for each of its containers (novox/hq
// ADR 0163, rule 4); judged above, so an invalid one is not here.
keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted)
// **A moved module's templated files are the provisioner's** (novox/hq ADR 0349 §4, §6, design 57 P6): on a
// machine that can take them, each is declared placed and without its text, and no pass below fills it.
handing, err := handed.begin(r.Node, m, dirs, accessPaths, thisMachine, with, r.Capabilities, r.Needs,
backupPlacer{append(append([]mcat.Manifest(nil), r.Modules...), stillBackedUp...), with, thisMachine, r.Capabilities,
unplaced})
if err != nil {
return nil, err
}
// **A plan for what the seat keys of it all the same** (catalogue.Keyed, issue 774): a module not handed whose
// provider's helper answers a provision of it, its route the commonest, has that provision in a plan of its own,
// so the seat keys it and the helper hears it.
keyedOnly := handing == nil && handed.keyed(r.Node, m, r.Needs)
if _, ok := handed.plans[m.Module]; ok && (handing != nil || keyedOnly) {
for _, to := range requirementsContributedTo(m) {
own, err := resolutionOwnNames(r, m, to, with.Settings[m.Module])
if err != nil {
return nil, err
}
handed.name(m.Module, to, own)
contributed, err := resolutionContributedOf(r, m, to, with)
if err != nil {
return nil, err
}
handed.contribute(m.Module, to, contributed)
}
}
// A handed collector is filled from items alone: each old form its collections would have held is said
// (design 57 L1).
if plan, ok := handed.plans[m.Module]; ok && handing != nil && len(plan.Collections) > 0 {
*unplaced = append(*unplaced, oldFormsLeftOut(append(append([]mcat.Manifest(nil), r.Modules...),
stillBackedUp...), m, plan.Collections)...)
}
for _, unsettled := range resources {
if pr, ok := unsettled[mcat.HandedProvisionField].(mcat.PlanProvision); ok {
if handing == nil && m.Stubbed {
continue // held: its credential file is what runs of it, left out (issue 1126)
}
if handing == nil {
return nil, fmt.Errorf("%s: its credential of %s is handed to the provisioner and the module is "+
"not: nothing would fill it", m.Module, pr.Provision)
}
marked := map[string]any{}
for k, v := range unsettled {
marked[k] = v
}
delete(marked, mcat.HandedProvisionField)
marked[mcat.HandedSecretField] = pr.Name
file, sec, err := handedSecretFile(marked, m, dirs, accessPaths, thisMachine, placed)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
pr.Path = sec.Path
if err := handed.addProvisionFile(m.Module, pr); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if unsettled[mcat.HandedSecretField] != nil {
if handing == nil && m.Stubbed {
continue // held: its own secret's file is what runs of it, left out (issue 1126)
}
if handing == nil {
return nil, fmt.Errorf("%s: its own secret %v is handed to the provisioner and the module is "+
"not: nothing would fill it", m.Module, unsettled[mcat.HandedSecretField])
}
file, sec, err := handedSecretFile(unsettled, m, dirs, accessPaths, thisMachine, placed)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
if err := handed.addSecretFile(m.Module, sec); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if handing != nil && fmt.Sprint(unsettled["type"]) == "file" {
if text, _ := unsettled["content"].(string); len(secretsUsed(text)) > 0 &&
!handing[fmt.Sprint(unsettled["id"])] {
return nil, fmt.Errorf("%s: its file %v names a secret in its text, and a handed module's "+
"secrets reach only the files its provisioner client fills: keep it in a template "+
"(design 57 V4)", m.Module, unsettled["id"])
}
}
if handing[fmt.Sprint(unsettled["id"])] {
file, line, err := handedFile(unsettled, m, dirs, accessPaths, thisMachine, placed, with)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
if err := handed.add(m.Module, line); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if fmt.Sprint(unsettled["type"]) == "file" {
if _, left := consumersHandedOnly(m, unsettled, with); left {
*handedOnly = append(*handedOnly, consumersLeftOutLine(m, unsettled))
continue
}
}
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
if err != nil {
return nil, err
}
copied := map[string]any{}
for k, v := range resource {
copied[k] = v
}
if networks, keeps := keptNetworks[fmt.Sprint(copied["id"])]; keeps {
// The container also joins the found network the setting names, so a neighbour
// that resolves it there keeps resolving it. Passed to the host as its own field,
// which it joins after the container is made.
joins := make([]any, 0, len(networks))
for _, n := range networks {
joins = append(joins, n)
}
copied["networks"] = joins
}
if err := refuseSecretsInEnvironment(copied, secretFiles, m.Module); err != nil {
return nil, err
}
// Said in the catalogue, not on the machine: the host parses strictly and knows no
// such field, and the reason is for a reader of the manifest.
delete(copied, mcat.SecretsInEnvironment)
delete(copied, mcat.NamesOnPurpose)
delete(copied, mcat.TrustedField)
// A process's bus stays controller-side (novox/hq ADR 0436 §1): never composed onto the
// node-engine's process resource, whose wire declaration is mesh-host's own, vendored, with a
// strict decoder that refuses an unknown key.
delete(copied, mcat.ProcessBusField)
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
// this node resolves for it, and every ${dir:…} — in paths, mounts, content and
// environment — becomes that path, so what follows sees only concrete places
// (novox/hq ADR 0112). The host receives paths exactly as it always has.
if err := dirInto(copied, dirs, m.Module); err != nil {
return nil, err
}
// The operator's data the same way: ${access:…} becomes where this node keeps it,
// and a placed directory takes the owner the assignment said (issue 153).
if err := accessInto(copied, accessPaths, m.Module); err != nil {
return nil, err
}
ownerInto(copied, placed)
// **After settings, and that is the whole reason it is here.** A module's file
// content is where a setting lands, so a placeholder may only exist once the setting
// has been put in — filling secrets first would look at content that is not yet what
// the machine receives.
if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil {
return nil, err
}
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
// holds it in the clear — so there is nothing for the host to be the only witness of,
// and sending it already filled in means the host learns no new field.
if err := boundInto(copied, known, m.Module, absent); err != nil {
return nil, err
}
// And what the module could not have written: the machine it turned out to be
// assigned to. Beside the bound values because it is the same kind of fact — the
// mesh's own, held in the clear — and because a module that must name itself to
// something else has no binding to learn it from (novox/hq ADR 0066).
// Which port this machine gave it, for a module that binds directly rather than
// through a runtime that can remap (ADR 0038). Applied before the machine's facts so
// a refusal names the port rather than whatever came after it.
if err := portInto(copied, m.Module, m.Listens, with); err != nil {
return nil, err
}
// And where this machine put the foundation's servers, for the one module that
// reaches them by seat rather than by binding (novox/hq 04-ISSUES/102).
if err := seatInto(copied, m.Module, with); err != nil {
return nil, err
}
if err := machineInto(copied, thisMachine, m.Module); err != nil {
return nil, err
}
if err := pinned(copied, m.Module); err != nil {
return nil, err
}
// And the other half of the same question: an image nobody has published yet is named
// by an artifact rather than by a placeholder digest, and is just as unrunnable.
if err := built(copied, m.Module); err != nil {
return nil, err
}
// What the mesh built is kept by digest and path; the store's address is this
// network's now, composed here and never recorded (novox/hq 04-ISSUES/102).
if err := artifactsInto(copied, m.Module, with); err != nil {
return nil, err
}
publishedOn(copied, m.Module, with)
// How it is ready, in the node-engine's words: its endpoint as the port this machine
// published it on — or not sent at all to an engine older than the field (ADR 0240).
healthInto(copied, m, with)
// And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it.
busyInto(copied, with)
// And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine
// that judges it.
rootInto(copied, with)
// The account's environment and every module's shell code, where this module holds the
// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
// the node, as the jails are, and **last of every placeholder pass**: shell code is a
// shell's own syntax, full of `${…}` no pass above should ever be shown.
contributing := append(append([]mcat.Manifest(nil), r.Modules...), stillBackedUp...)
if err := contributionsInto(copied, m, contributing, thisMachine, with, r.Capabilities, unplaced); err != nil {
return nil, err
}
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
// A service saying what it reflects names resources within its own module, so those
// are prefixed too or they would point at nothing.
//
// **Unless it already names one.** A module may reflect a file another module put on
// the machine — the case this exists for is a resolver restarting when the mesh
// rewrites the names, which are computed by the mesh and belong to its module, not to
// the daemon's. Written `<module>.<id>`, and a dot is what marks it as already
// answered: prefixing it again would point at nothing, silently, and the daemon would
// serve the old names for ever while everything reported success.
if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil {
copied["restart-on"] = renamed
}
// And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left
// unprefixed matches nothing, and the service is never reloaded.
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
copied["reload-on"] = renamed
}
// And which of its module's containers a scheduled step holds still (novox/hq ADR 0189).
// **The loudest of the three when it is missed.** An unprefixed `restart-on` matches
// nothing and a service quietly never restarts; an unprefixed `while-stopped` names a
// container the declaration does not contain, and the host refuses the whole
// declaration — so the machine takes nothing at all, for every push, until this is
// right. That is what it did on the control node (2026-10-04).
if renamed := reflectsRenamed(m.Module, resource[mcat.WhileStopped]); renamed != nil {
copied[mcat.WhileStopped] = renamed
}
// And what a process replaces (novox/hq issue 213): a resource of this module's that it
// no longer declares, named as the host recorded it, or the host hands nothing over and
// removes it first.
if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil {
copied["replaces"] = renamed
}
// **What reads one of this module's own secrets is restarted when it changes** (novox/hq
// issue 203, issue 206). A credential is re-issued by the mesh, and a container that
// mounted the old file keeps the old one open: the build machine ran for an hour on a
// credential the mesh had replaced, because its manifest restarted it on its
// environment file and nobody had thought to name the credential too. Composed here so
// no manifest has to say it, for a container or a daemon that names the secret's path.
if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 {
copied["restart-on"] = withRestartOn(copied["restart-on"], reads)
}
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
// module's own resource rather than declared beside it: what prepares the state is the
// module's own code, so what it is given has to be what that code is given — and a
// second resource written by hand is a second copy to drift from the first. Placed
// immediately before it, because a run-once step stops everything the declaration
// places after it (ADR 0052), which is how a version whose preparation failed does not
// serve.
if prepareBefore != "" && fmt.Sprint(resource["id"]) == prepareBefore {
step := prepared(copied)
owner[fmt.Sprint(step["id"])] = m.Module
out = append(out, step)
}
owner[fmt.Sprint(copied["id"])] = m.Module
out = append(out, copied)
}
// **What only the mesh knows, where this module asked for it.** The graph is the control
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
// this module's name, so they are applied, reported and removed exactly as anything else
// it declares.
// The resolver file is the node-resolver holder's where one is held here, and the uplink's
// holder steps back from it (novox/hq ADR 0247).
given, err := FactsFrom(stepsBack(m, r.Modules), r, with)
if err != nil {
return nil, err
}
for _, fact := range given {
fact["id"] = m.Module + "." + fmt.Sprint(fact["id"])
owner[fmt.Sprint(fact["id"])] = m.Module
out = append(out, fact)
}
}
// The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every
// bundle delivered above and holding the credential sealed above, so both exist before it starts
// — the order written here is the order the machine applies.
if resolutionRuntimeHere(r) {
process, err := resolutionRuntimeProcess(r, with)
if err != nil {
return nil, err
}
if process != nil { // none for a held runtime's stand-in: what runs of it is carried
owner[fmt.Sprint(process["id"])] = fmt.Sprint(process["name"]) // the tool runner's module (resolutionRuntimeProcess)
out = append(out, process)
}
// And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8).
owns, err := resolutionOwnRuntimes(r, with)
if err != nil {
return nil, err
}
for _, p := range owns {
id := fmt.Sprint(p["id"])
owner[id] = strings.TrimSuffix(id, "."+mcat.OwnRuntimeID())
out = append(out, p)
}
// What each module's bundles are given is read as the account the runtime runs as — not what a
// module of its own account is given, which its own account reads.
words := map[string]map[string]string{}
for _, m := range r.Modules {
if m.RunsAs != "" {
continue
}
w, err := bundleWords(m, with)
if err != nil {
return nil, err
}
words[m.Module] = w
}
// And a file a module's words name is one the runtime is restarted for when it changes.
if named := givenTo(out, owner, words, r.Account); len(named) > 0 && process != nil {
restarts, _ := process["restart-on"].([]any)
seen := map[string]bool{}
for _, id := range restarts {
seen[fmt.Sprint(id)] = true
}
for _, id := range named {
if !seen[id] {
restarts = append(restarts, id)
seen[id] = true
}
}
process["restart-on"] = restarts
}
// **And what a module of its own account is given, its own account reads** (issue 723): the files and
// directories its words name that say no owner, given to the account it runs as, never the operator's.
for _, m := range r.Modules {
if m.RunsAs == "" {
continue
}
w, err := bundleWords(m, with)
if err != nil {
return nil, err
}
givenTo(out, owner, map[string]map[string]string{m.Module: w}, m.RunsAs)
}
}
if with.Adopted {
// First, before anything a module declares: what the mesh needs reachable, then its guard.
// The order a machine applies is the order written here.
ours := Openings(rules, with.Foundation, mcat.Published(out))
ours = append(ours, GuardResources(resolutionGuarded(r, out, owner, rules, with))...)
out = append(ours, out...)
}
return out, nil
}
// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and
// for taken modules only.
//
// For each taken module, every machine port its containers publish that the filter would admit
// from the private network only — a published port is forwarded, not received, so a found
// firewall filtering only what it receives never sees it — together with the ports the module's
// manifest guards explicitly (the store's port, the broker's management port), wherever the
// machine put them. A port of a module assigned but not taken is not guarded: it may still be the
// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted
// from everywhere and are never guarded.
func resolutionGuarded(r mcat.Resolution, out []map[string]any, owner map[string]string, rules []mcat.Rule,
with mcat.Rendering) []int {
meshOnly := map[int]bool{}
fromEverywhere := map[int]bool{}
for _, rule := range rules {
if rule.Protocol != "tcp" {
continue
}
switch rule.From {
case mcat.FromMesh:
meshOnly[rule.Port] = true
case mcat.FromEverywhere:
fromEverywhere[rule.Port] = true
}
}
for _, port := range with.Foundation {
delete(meshOnly, port)
fromEverywhere[port] = true
}
seen := map[int]bool{}
var ports []int
guard := func(at int) {
if !seen[at] {
seen[at] = true
ports = append(ports, at)
}
}
for _, m := range r.Modules {
if !with.Taken[m.Module] {
continue
}
var mine []map[string]any
for _, resource := range out {
if owner[fmt.Sprint(resource["id"])] == m.Module {
mine = append(mine, resource)
}
}
for outer := range mcat.Published(mine)["tcp"] {
if meshOnly[outer] {
guard(outer)
}
}
for _, want := range m.Guards {
at := renderingMachinePort(with, m.Module, want)
for _, resource := range mine {
if fmt.Sprint(resource["type"]) != "container" {
continue
}
listed, _ := resource["ports"].([]any)
for _, entry := range listed {
outer, inner, _, ok := mapping(fmt.Sprint(entry))
if ok && inner == want {
at = outer
}
}
}
// Not what this node is told to open to everyone: a per-node exposure setting that
// widens a guarded port is the operator saying so, and declaring an opening for it
// and a guard dropping it would be one statement refusing the other.
if !fromEverywhere[at] {
guard(at)
}
}
}
sort.Ints(ports)
return ports
}
// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address
// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never
// shift the ports. Not ok for a short form or anything that is not a mapping.
func mapping(written string) (outer, inner int, address string, ok bool) {
written = strings.TrimSpace(written)
if cut := strings.LastIndex(written, "/"); cut >= 0 {
written = written[:cut]
}
parts := strings.Split(written, ":")
if len(parts) < 2 {
return 0, 0, "", false
}
inner, err := strconv.Atoi(parts[len(parts)-1])
if err != nil {
return 0, 0, "", false
}
outer, err = strconv.Atoi(parts[len(parts)-2])
if err != nil {
return 0, 0, "", false
}
return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true
}
// filtersHere is the module on this node that loads the machine's packet filter, or empty when none
// does. Named rather than counted: a refusal that says which module is one step from acted on.
func resolutionFiltersHere(r mcat.Resolution) string {
for _, m := range r.Modules {
if m.Filtering != nil {
return m.Module
}
}
return ""
}
// Rules is the rule set this node's filter is derived from: every module's listens, what was
// computed for this machine, and each module's per-node exposure. The same answer whether the node
// is adopted or converged — the one loads it as a filter, the other declares it as openings.
func Rules(r mcat.Resolution, with mcat.Rendering) ([]mcat.Rule, error) {
// A module whose settings cannot compose is left out of the declaration (novox/hq ADR 0163,
// rule 6), and out of the filter with it: nothing of it is declared, so nothing of it is let
// through.
left := LeftOut(r, with.Settings, with.Adopted)
exposure := map[string]map[int]string{}
for _, m := range r.Modules {
if _, isLeft := left[m.Module]; isLeft {
continue
}
e, err := Exposure(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
// And how far each endpoint reaches, which says the same thing to the filter and more
// besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one
// question — from where — and a reach answers it, so giving it two inputs would let them
// disagree. Reaches refuses a port that both name, so this cannot silently prefer one.
reaches, err := Reaches(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
// **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the
// proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and
// says nothing about the port — opening it to the world as well would undo the arrangement
// the proxy exists for, and would silently reopen a port an operator had narrowed.
//
// Found by trying to express a real module: one whose routed name must be public and whose
// machine-side port must not be. Under one value for both, there was no way to say it.
routed := RoutedPorts(m)
for port, reach := range reaches {
if routed[port] {
continue
}
source, ok := FilterSource(reach)
if !ok {
return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach)
}
if e == nil {
e = map[int]string{}
}
e[port] = source
}
if e != nil {
exposure[m.Module] = e
}
}
return Filtering(r, with.Generators, with.Ports, exposure)
}
// provisionsAs is the account that reads what the mesh writes for this provider: the one secret
// per consumer it must open to set that consumer's password (novox/hq issue 225).
//
// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already
// written above for a module's own secrets — and the grant secret is the other kind of secret
// the mesh writes for a module, so it is the same rule.
//
// Which account depends on where the module's code runs. A module whose code is a bundle is run
// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container
// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that
// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the
// harness composes a grant secret's path from the contributions file, not from a word.
//
// Empty is root, which is what it was and what a module with no bundle and no declared owner
// still wants.
func resolutionProvisionsAs(r mcat.Resolution, m mcat.Manifest) string {
// A module of its own account reads them as that account (ADR 0259 §8, issue 723).
if m.RunsAs != "" {
return m.RunsAs
}
if len(m.Bundles) > 0 && r.Account != "" {
return r.Account
}
return m.SecretsOwner
}
// ownSecretOwner is who one of a module's own secret files belongs to: the account the manifest names for it, else the
// runtime's account for the runtime's own credential and for a tools bundle's bus account (ToolsBusSecret; novox/hq ADR
// 0564 Limits 4), which the runtime runs that bundle as, else the module's secrets-owner (root when none).
func resolutionOwnSecretOwner(r mcat.Resolution, m mcat.Manifest, name string) string {
if o := m.OwnSecrets[name].Owner; o != "" {
return o
}
if r.Account != "" && (mcat.IsToolRunner(m) || ToolsBusSecret(m, name)) {
return r.Account
}
return m.SecretsOwner
}
// grantPath is where one consumer's sealed credential lands on the providing machine.
//
// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a
// node named like something else in that directory cannot collide with it.
// One file per consumer, and a consumer is a module on a machine (novox/hq 04-ISSUES/022).
//
// Named after both. Named after the machine alone, two modules on one node wrote to one path: the
// second overwrote the first, and the provisioner — reading a directory — saw one consumer where
// there were two.
// reflectsRenamed is a resource's restart-on list under the module's prefix, or nil when it has
// none. It reads both the shape JSON parsing produces ([]any) and the shape code composing
// resources natively produces ([]string): a reference that was skipped because its list arrived
// in the other shape would point at nothing — silently, with the service never restarting and
// every check passing, which is how a runtime kept serving without the registry trust its
// daemon file already carried.
func reflectsRenamed(module string, reflects any) []any {
var names []string
switch v := reflects.(type) {
case []any:
for _, id := range v {
names = append(names, fmt.Sprint(id))
}
case []string:
names = v
}
if names == nil {
return nil
}
var renamed []any
for _, named := range names {
if strings.Contains(named, ".") {
renamed = append(renamed, named)
continue
}
renamed = append(renamed, module+"."+named)
}
return renamed
}
func grantPath(directory, consumer, module string) string {
return strings.TrimRight(directory, "/") + "/" + consumer + "." + module + ".secret"
}
// holderAs is a consumer's name at the provider with a local name after it, where it keeps several
// credentials for one provision (ADR 0094); the name alone otherwise.
func holderAs(as, local string) string {
if local == "" {
return as
}
return as + "_" + local
}
// contributions collects what every module in this set contributes, by requirement.
//
// Ordered by contributing module, because the result becomes a file on a machine and a file whose
// lines move about is a file that looks changed when nothing changed.
func resolutionContributions(r mcat.Resolution, settings mcat.SettingsBy, grants []mcat.Grant,
directories map[string]string) (map[string][]mcat.Contribution, error) {
out := map[string][]mcat.Contribution{}
modules := append([]mcat.Manifest{}, r.Modules...)
sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module })
// What consumers on other machines asked for. Merged in with this machine's own, because from
// the provider's side they are the same thing — somebody wanting something — and a provider
// that had to read two lists would be a provider that reads one of them.
sorted := append([]mcat.Grant{}, grants...)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Provision != sorted[j].Provision {
return sorted[i].Provision < sorted[j].Provision
}
if sorted[i].Consumer != sorted[j].Consumer {
return sorted[i].Consumer < sorted[j].Consumer
}
if sorted[i].From != sorted[j].From {
return sorted[i].From < sorted[j].From
}
return sorted[i].Local < sorted[j].Local
})
// A consumer already carried by the grants loop, keyed (provision, module). When provider and
// consumer are co-located, `grantsFor` enumerates the same-node consumer too, so without this the
// module would be emitted a second time by the m.Contributes loop below — once full (with the
// grant's secret/as) and once partial — which is the duplicate seen in a co-located mesh.json.
granted := map[string]map[string]bool{}
for _, g := range sorted {
if g.From == "" {
// As above: nothing on that machine asks for this any more, so the provider is not
// told about it and withdraws the login on its next pass.
continue
}
as := holderAs(mcat.ConsumerIdentity(g.Consumer, mcat.IdentitySource(g.Slug, g.From)), g.Local)
derived, err := resolutionDerivedFor(r, g.Provision, as, g.From, g.Local, settings)
if err != nil {
return nil, err
}
out[g.Provision] = append(out[g.Provision], mcat.Contribution{
From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived,
// One holder per local name: the identity the consumer is known by, and the local name
// after it where the module keeps several (ADR 0094). Not a login any backend checks —
// a secret is not a login — so the identity limit does not apply to the suffix.
As: as,
Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)),
})
if granted[g.Provision] == nil {
granted[g.Provision] = map[string]bool{}
}
granted[g.Provision][g.From] = true
}
for _, m := range modules {
for _, to := range sortedKeys(m.Contributes) {
// The grants loop already emitted this module's contribution to this provision, with its
// minted secret — emitting the partial copy again would duplicate it. A contribution with
// no grant (a reverse-proxy route names a host, not a credential) is not in `granted`, so
// it still reaches the provider from here.
if granted[to][m.Module] {
continue
}
// Settings reach a contribution the same way they reach a file. A route's hostname is
// exactly the kind of thing that differs between one mesh and the next, and a module
// that could not have it set would have to be edited to be reused.
values, err := resolutionComposed(r, m, to, m.Contributes[to], settings[m.Module],
m.Module+" contributing to "+to)
if err != nil {
return nil, err
}
// What the module shares of its bucket reaches the provider with the rest (novox/hq ADR 0468 §8).
if values, err = withShares(m, to, values); err != nil {
return nil, err
}
out[to] = append(out[to], mcat.Contribution{From: m.Module, Values: values})
}
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
// object store's data API and its console are two different public names from one module,
// not one. Never in `granted` — a route names a host, not a credential — so every local
// name always reaches the provider from here.
for _, to := range sortedKeys(m.ContributesMany) {
for _, local := range sortedKeys(m.ContributesMany[to]) {
values, err := resolutionComposed(r, m, to, m.ContributesMany[to][local], settings[m.Module],
m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, err
}
out[to] = append(out[to], mcat.Contribution{From: m.Module, Values: values})
}
}
}
return out, nil
}
// contributedOf is each contribution m makes to one requirement, composed as its provider receives it in a received
// file (contributions: the one of `contributes` with what the module shares of its bucket, then each of several by
// local name), with where m's machine is, encoded for PlanProvision.Contributed (issue 774): one composition, so a
// plan and a file cannot say different things. Nil where m contributes nothing to it.
//
// **Its port is the one the machine published** (atAssignedPort): a plan is read on whatever machine its provider runs,
// which cannot redirect the number as a received file on this machine is redirected (atMachinePort), so the plan
// carries the machine port already. route-proxy dialled gitea's 3000 for git (published at 20000) and grafana's 3000
// on another machine (published at 20012) until it did (2026-10-31).
func resolutionContributedOf(r mcat.Resolution, m mcat.Manifest, to string, with mcat.Rendering) (json.RawMessage, error) {
layers := with.Settings[m.Module]
form := mcat.ContributedForm{At: r.At}
add := func(local string, values map[string]any) error {
values, err := resolutionAtAssignedPort(r, m, to, values, with)
if err != nil {
return err
}
raw, err := json.Marshal(values)
if err != nil {
return fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
form.Each = append(form.Each, mcat.ContributedOne{Local: local, Values: raw})
return nil
}
if raw, ok := m.Contributes[to]; ok {
values, err := resolutionComposed(r, m, to, raw, layers, m.Module+" contributing to "+to)
if err != nil {
return nil, err
}
if values, err = withShares(m, to, values); err != nil {
return nil, err
}
if err := add("", values); err != nil {
return nil, err
}
}
for _, local := range sortedKeys(m.ContributesMany[to]) {
values, err := resolutionComposed(r, m, to, m.ContributesMany[to][local], layers, m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, err
}
if err := add(local, values); err != nil {
return nil, err
}
}
if len(form.Each) == 0 {
return nil, nil
}
return EncodeContributed(form)
}
// atAssignedPort is a contribution of m with its port where this machine publishes it for m: the number
// `${port:<software port>}` gives m itself and its declaration publishes (machinePort, port_into.go), the placement's
// assignment where it holds one and the software's own number where it holds none (a bare port not yet moved, a fixed
// mapping, the host's network). What its provider dials, on this machine over loopback and from another at m's address.
//
// **Refused where a plan carries a port m does not listen on** (with.Planned): no machine port stands for it, so the
// provider would dial a number nothing on the machine is known to publish. Elsewhere (a declaration, a composition
// with no ports) such a port is left as it is, as machinePort leaves it.
func resolutionAtAssignedPort(r mcat.Resolution, m mcat.Manifest, to string, values map[string]any, with mcat.Rendering) (map[string]any, error) {
number, ok := asPort(values["port"])
if !ok {
return values, nil
}
_, given := with.Given[m.Module][number]
_, placed := with.Ports[m.Module][number]
if !given && !placed {
if with.Planned {
return nil, fmt.Errorf("%s contributing to %s: its port %d on %s is none of the ports it listens on, so no "+
"machine port stands for it", m.Module, to, number, r.Node)
}
return values, nil
}
return atMachinePort(values, m.Module, map[string]map[int]int{m.Module: {number: renderingMachinePort(with, m.Module, number)}}), nil
}
// composed is one contribution as its provider receives it: settled with this node's settings, its
// endpoint's port filled in, and its names composed from the label.
//
// **One function, because two readers must agree.** The provider is told the names in its received
// file; the contributing module is told the same names in its own binding (novox/hq 04-ISSUES/122).
// Composing them twice, in two places, is how the proxy would come to serve one name while the
// module wrote another into its configuration.
func resolutionComposed(r mcat.Resolution, m mcat.Manifest, to string, raw map[string]any, layers []mcat.Layer, what string) (
map[string]any, error) {
// Overridden, not merged: a setting changes a key the contribution declares and adds none.
// The provider reads the contribution as a contract, and a setting made for one of this
// module's files is no part of it (novox/hq 04-ISSUES/173).
values, err := overridden(raw, WithDefaults(m, layers), what)
if err != nil {
// A required value nothing sets is said in the words of the module's form, as a file's, a unit's and an owner's
// are (issue 1126, seed-8: de-spiegel in the config form was refused with ${setting:…} and `settings set`).
var unset *UnsetSettingError
if errors.As(err, &unset) && unset.Module == "" {
unset.Module = m.Module
unset.where = fmt.Sprintf("a contribution%s, in %q,", strings.TrimPrefix(what, m.Module+" contributing"), unset.field)
}
return nil, unsetInConfigWords(fmt.Errorf("%s: %w", what, err), m)
}
reaches, err := Reaches(m, layers)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
blocks, err := mcat.Endpoints(m, layers)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
portOfEndpoint(values, endpointPorts(m))
composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks)
return values, nil
}
// ownNames is what a module is known by through what it contributes to one requirement — the names
// the mesh composed for it, and nothing else of the contribution.
//
// **The half a module could not learn** (novox/hq 04-ISSUES/122). A module contributes a label, the
// mesh joins it with this node's domains, and the provider serves the result — and the module itself
// was never told. Software that must know its own address (a login redirect, a canonical URL, an
// issuer) had it written into the manifest as a literal, which is a domain in a definition and wrong
// on every other machine. `${bound:<requirement>:name}` is the answer, from the same composition the
// provider receives.
//
// Several contributions to one requirement are keyed by their local name under `names`.
func resolutionOwnNames(r mcat.Resolution, m mcat.Manifest, to string, layers []mcat.Layer) (map[string]any, error) {
pick := func(values map[string]any) map[string]any {
names := map[string]any{}
for _, key := range []string{"name", "internal-name"} {
if v, ok := values[key].(string); ok && v != "" {
names[key] = v
}
}
return names
}
out := map[string]any{}
if raw, ok := m.Contributes[to]; ok {
values, err := resolutionComposed(r, m, to, raw, layers, m.Module+" contributing to "+to)
if err != nil {
return nil, err
}
for k, v := range pick(values) {
out[k] = v
}
}
if locals := m.ContributesMany[to]; len(locals) > 0 {
many := map[string]any{}
for _, local := range sortedKeys(locals) {
values, err := resolutionComposed(r, m, to, locals[local], layers,
m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, err
}
if names := pick(values); len(names) > 0 {
many[local] = names
}
}
if len(many) > 0 {
out["names"] = many
}
}
return out, nil
}
// composeName joins a contribution's label with a node's public domain, and separately with its
// private one, in place (novox/hq ADR 0056).
//
// **The whole of what the mesh does with a route's name: join two given strings — twice.** A
// contribution carries a `label` — the subdomain its operator chose — and the node carries its
// public domain and its own private-network address; the granted names are `<label>.<public-domain>`
// and `<label>.<internal-domain>`, and the mesh interprets none of the halves. It runs on any
// contribution carrying a label, not only a route's, because the mesh does not know what a
// provision means — a name it can compose from parts it was given is the point, whatever the
// provision is called.
//
// **The internal name is not a security boundary.** A predecessor proxy that answered both a
// public and a private-network hostname for the same route did so as a convenience — reaching a
// service over the VPN without a public TLS round trip — not as an access control, and composing
// the same alias here restores that convenience rather than adding one. A route with no internal
// domain to compose against (a node not on the private network) gets no internal name, the same as
// it gets no public one with no public domain.
//
// **Additive, so an unmigrated catalogue still works.** A contribution that already carries a full
// `name` and no `label` is left exactly as it is: the catalogue can migrate module by module while
// the running mesh keeps serving the full names it has. And a labelled contribution on a node with
// no public domain composes nothing — there is nothing to join it to — which reads downstream as a
// route that named no host, the same as it would have before this existed.
func composeName(values map[string]any, publicDomain, internalDomain string, reaches map[int]string,
ports map[string]int, blocks map[string]mcat.Endpoint) {
if values == nil {
return
}
// **The subdomain an assignment gave this endpoint**, before the name is joined (novox/hq ADR
// 0138). The module contributes a label because it names its own parts; an assignment may say a
// different one, because where a thing lives under a domain is the operator's to choose and used
// to require editing the module to change.
if name, ok := values[mcat.RouteEndpoint].(string); ok {
if ep, said := blocks[strings.TrimSpace(name)]; said && ep.Label != "" {
values["label"] = ep.Label
}
}
// **How far the endpoint this route serves reaches decides which names exist** (novox/hq ADR
// 0138). Both were composed whenever the node had both domains, so every routed module got a
// public name and an internal one whether anybody wanted them or not — and a certificate for
// each, because the proxy certifies the names it is given.
//
// Joined by the port: a route entry names the port it serves and the module declares a listen on
// it. An entry with no port is not an endpoint's route but a rule about a name — a path-level
// refusal shadowing another route — and it inherits whatever that route's names turned out to
// be, which is why it is left alone here.
//
// Nothing said is both names, as before. That is what keeps every mesh already running identical
// until an assignment speaks.
wantPublic, wantInternal := true, true
if port, ok := endpointPortOf(values, ports); ok {
if reach, said := reaches[port]; said {
wantPublic, wantInternal = WantsPublicName(reach), WantsInternalName(reach)
}
}
if !wantPublic {
publicDomain = ""
}
if !wantInternal {
internalDomain = ""
}
if _, already := values["name"]; already {
// A full name was given rather than a label. Left as-is: this is the legacy shape, and the
// point of the label is to not have to write the full name — a contribution that wrote both
// has said what it wants and the mesh does not second-guess it.
return
}
label, ok := values["label"].(string)
if !ok || strings.TrimSpace(label) == "" {
return
}
trimmed := strings.TrimSpace(label)
if trimmed == "@" {
// The apex: a module served at the bare domain, no subdomain — the zone-file convention
// `@`. Composes to the domain itself, so a node's own site is a label like any other rather
// than the one route that must still carry a full name.
if publicDomain != "" {
values["name"] = publicDomain
}
if internalDomain != "" {
values["internal-name"] = internalDomain
}
return
}
if publicDomain != "" {
values["name"] = trimmed + "." + publicDomain
}
if internalDomain != "" {
values["internal-name"] = trimmed + "." + internalDomain
}
}
// servingAt is the private-network name of the node a contribution to `to` arrives at: the
// provider's, when the provision is answered elsewhere, and this machine's own when it is answered
// here or not yet settled.
//
// A route's internal name is composed under it (novox/hq ADR 0151, issue 139). `<label>.<node>.internal`
// is answered by every machine's resolver as *anything under that node's name goes to that node* —
// so the node in the name has to be the one whose proxy answers, or the name sends a client to a
// machine with nothing listening while the public name, published at the serving node's address,
// works. Where the proxy runs beside the module the two are the same machine and nothing changes.
func servingAt(r mcat.Resolution, to string) string {
for _, n := range r.Needs {
if n.Name == to && n.At != "" {
return n.At
}
}
return r.At
}
// receivedFile is the file a provider is given its consumers' contributions in.
func receivedFile(requirement, path string, given []mcat.Contribution) (map[string]any, error) {
if given == nil {
// Nobody contributed. The file is still written, empty, rather than left absent: a
// provider that finds no file cannot tell "nothing asked for me" from "the mesh never
// wrote it", and the two want completely different responses.
given = []mcat.Contribution{}
}
// The note goes *inside* the document, not above it. The first version wrote a `//` header
// and produced a file that says "do not edit" to a person and fails to parse for the program
// meant to read it — which is the whole audience.
body, err := json.MarshalIndent(map[string]any{
"contributions": 1,
"requirement": requirement,
"generated": "by the mesh — do not edit; replaced whenever a module contributing to " +
requirement + " arrives or leaves",
"given": given,
}, "", " ")
if err != nil {
return nil, err
}
return map[string]any{
"id": mcat.ReceivedID(requirement), "type": "file", "path": path, "mode": "0644",
"content": string(body) + "\n",
}, nil
}
// KeptID is the resource that carries the export to a module that keeps it.
func KeptID() string { return "kept" }
// keptFile is the export, written where the module said. Ciphertext throughout — see Keeps.
func keptFile(dir string, kept *mcat.KeptExport) (map[string]any, error) {
body, err := json.MarshalIndent(kept, "", " ")
if err != nil {
return nil, err
}
return map[string]any{
"id": KeptID(), "type": "file", "path": strings.TrimRight(dir, "/") + "/export.json",
"mode": "0600", "content": string(body) + "\n",
}, nil
}
// ownedBy gives a secret file the owner the module named, when it named one (Manifest.SecretsOwner).
func ownedBy(owner string, file map[string]any) map[string]any {
if owner != "" {
file["owner"] = owner
}
return file
}
// sortedKeys is map iteration made repeatable, which everything written to a machine needs.
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// boundFile is what a module is told about something it requires from another machine.
//
// Where it is and what the providing module said about using it. **No credential**, and the file
// says so rather than leaving a reader to wonder whether one was meant to be there — a missing
// field looks like a bug, and a stated absence looks like a boundary.
func boundFile(n mcat.Needed, path, as string, own map[string]any) (map[string]any, error) {
// A record has no machine and no address. Saying so is the difference between a reader
// concluding "somewhere with no address" and concluding the mesh failed to fill something in.
where := any(n.At)
if n.ByRecord {
where = "a record in this mesh, not a machine"
}
doc := map[string]any{
"binding": 1,
"provision": n.Name,
"from": n.From,
"at": where,
// Who this module is at the other end. **The half that was missing** (novox/hq
// 04-ISSUES/023): a consumer was told the address, the port and where its password is,
// and not the name it must present — which the provisioner had invented.
"as": as,
"serves": n.Serves,
// **Where the credential is, not what it is.** It stopped being true that the mesh
// cannot issue one when 021 was fixed, and a comment asserting a fact about the mesh that
// has become false is worse than none — somebody reads it and stops looking.
"generated": "by the mesh — do not edit; replaced whenever this changes. " +
"The credential is not here: it is sealed, in the file this module's manifest " +
"names under `secrets`",
}
// **What this module is called through what it contributes here** (novox/hq 04-ISSUES/122):
// `name`, `internal-name`, or `names` by local name — composed exactly as the provider receives
// them. Absent when the module contributes nothing named, rather than written empty.
for key, value := range own {
doc[key] = value
}
body, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return nil, err
}
return map[string]any{
"id": mcat.BoundID(n.Name), "type": "file", "path": path, "mode": "0644",
"content": string(body) + "\n",
}, nil
}
// ContributionsFrom is what one module on this node asked of one requirement, settled.
//
// Exported because a provider's grants are assembled from its consumers' resolutions, one machine
// at a time, and the alternative was for the control plane to reimplement settling.
//
// **One module, not one machine** (novox/hq 04-ISSUES/022). This used to take a requirement alone
// and refuse whenever two modules wanted it — correctly, given what it had: the credential was
// keyed by node, so the two would have shared one, and sharing is worse than refusing. But the
// arrangement refused is the ordinary one. A node running eight services against one database is
// not an edge case; it is what a machine looks like. Now each consumer has its own credential and
// there is nothing left to refuse.
//
// **One credential, even where a module contributes several times.** A module may answer one
// requirement more than once (ADR 0094's sibling for `contributes`) — an object store's data API
// and its console are two different names, not one. There is still only one `Needed` for it, one
// credential minted, one grant to settle: a pair credential is not a place to put a label or a
// port. So where several of this module's contributions reach the same requirement, none of them
// is "the" value — settling to the first, arbitrarily, would hand the grant one contribution's
// values under a credential the OTHER contribution's consumer never sees, and would collide with
// that contribution's own entry from contributions() besides. Empty values, still granted: the
// module asked, gets its credential, and each named contribution reaches the provider on its own.
func ContributionsFrom(r mcat.Resolution, requirement, module string, settings mcat.SettingsBy) (
map[string]any, bool, error) {
all, err := resolutionContributions(r, settings, nil, nil)
if err != nil {
return nil, false, err
}
var mine []map[string]any
for _, g := range all[requirement] {
if g.From == module {
mine = append(mine, g.Values)
}
}
var asker mcat.Manifest
for _, m := range r.Modules {
if m.Module == module {
asker = m
}
}
if len(mine) == 1 {
// Its shares are already in, put there where its contribution was composed (contributions).
return mine[0], true, nil
}
if len(mine) > 1 {
values, err := withShares(asker, requirement, map[string]any{})
return values, err == nil, err
}
// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
// granted a credential. Keying "asks" on contributions alone marked those grants withdrawn
// (From=""), so the provider never created the account and the consumer authenticated nowhere. A
// module asks iff it still requires the provision, whether or not it hands anything up with it.
for _, m := range r.Modules {
if m.Module != module {
continue
}
for _, req := range m.Requires {
if req == requirement {
values, err := withShares(m, requirement, map[string]any{})
return values, err == nil, err
}
}
}
// Nothing on that machine asks for this any more. Said as "not found" rather than as an
// error: it is how a credential is withdrawn, and the provider removes what nobody asks for.
return nil, false, nil
}
// here is the module on this same machine that answers a requirement, as a binding.
//
// **Only when the provider said something a consumer must know.** That is the line: the original
// reasoning — a file saying "it is on this node" is a fact nobody needs — is right about the
// location and wrong about everything beside it. A shell is answered here and there is nothing to
// say about it. A registry is answered here and the consumer still cannot guess the port.
//
// The address is this machine's own name on the private network when it has one, and loopback
// when it does not — a machine not on the network still reaches itself, and naming it by a name
// nothing resolves would be worse than naming it by an address that always works.
func here(r mcat.Resolution, requirement string, with mcat.Rendering) (*mcat.Needed, error) {
serves, answered, err := resolutionServedOnThisMachine(r, requirement, with)
if err != nil || !answered {
return nil, err
}
at := r.At
if at == "" {
at = "127.0.0.1"
}
return &mcat.Needed{Name: requirement, From: r.Node, At: at, Serves: serves}, nil
}
// servedOnThisMachine is what a provider in this node's own set says a consumer must know —
// derived exactly as the mesh derives it for a provider on any OTHER machine.
//
// **The one derivation, so the two arrangements cannot disagree.** For a provider elsewhere the
// control plane walks that node, reads its manifest with that machine's port assignments, and
// settles the result with that node's settings layers before offering it (cmd/mesh-controller plan.go,
// theRestOfTheMesh). A provider on the consumer's own machine never passes through that walk, so
// every step of it has to be repeated here — and each step that was not repeated was a promise the
// co-located arrangement quietly broke: first the port (novox/hq 04-ISSUES/038), then the settled
// values (ADR 0066, an internal CA's root arriving empty).
//
// Here rather than in the resolver, because here is the first moment both halves exist: resolving
// runs before a machine's ports are assigned and knows nothing of settings.
//
// The first module in the resolved order that says it serves the provision answers, which is the
// same choice here() has always made and is stable — providersFirst orders the slice, and nothing
// downstream depends on a map's iteration.
func resolutionServedOnThisMachine(r mcat.Resolution, provision string, with mcat.Rendering) (map[string]any, bool, error) {
for _, m := range r.Modules {
if _, said := m.Serves[provision]; !said {
continue
}
serves := ServedOn(m, provision, with.Ports[m.Module])
if len(serves) == 0 {
// It named the provision and said nothing about it, which is the same as not answering:
// there is no fact for a consumer to be given. Another module in the set may still have
// one, so keep looking rather than concluding from the first.
continue
}
settled, err := Settle(serves, WithDefaults(m, with.Settings[m.Module]))
if err != nil {
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
// And the provider's own names, as this machine composes them (novox/hq ADR 0346); one it does
// not compose leaves its key out, said by the plan on this machine (ServedLeftOut).
named, _, err := ServedOwnNames(r, m, provision, settled, WithDefaults(m, with.Settings[m.Module]))
if err != nil {
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
return named, true, nil
}
return nil, false, nil
}
// pinned refuses an image that is not really pinned, on its way to a machine.
//
// **Here and not at parse** (novox/hq 04-ISSUES/025). A manifest in a repository names artifacts
// and the manifest the mesh holds names digests — they are deliberately not the same document, so
// a file awaiting a pin is legitimate exactly as the bundle's is. What must never happen is a
// placeholder reaching a machine, and this is the last moment before one does.
//
// The host checks only the *shape* of a reference, and cannot do more: verifying a digest exists
// means reaching a registry, which is the one thing a host must never have to do. So sixty-four
// zeros satisfies every gate in the system and stops on the machine at `docker pull` — which is
// how eighteen of them shipped across five modules that parse, resolve and compose cleanly.
func pinned(resource map[string]any, module string) error {
image, ok := resource["image"].(string)
if !ok {
return nil
}
_, digest, found := strings.Cut(image, "@")
if !found || strings.Trim(strings.TrimPrefix(digest, "sha256:"), "0") != "" {
return nil
}
return fmt.Errorf(
"%s would send %v to a machine pinned to a placeholder digest, which is never a real "+
"image — it would be fetched and fail there. Resolve the tag to a digest first",
module, resource["id"])
}
// built refuses a container that still names an artifact nobody has made.
//
// **Said here, by the thing that knows what "artifact" means.** A container naming an artifact is
// a module saying "the mesh builds this"; the field is resolved into an image when a build
// publishes one, and until then there is nothing to run. A host receiving it refuses the whole
// declaration — correctly, since its language has no such field — but what it can say is that a
// container does not use "artifact", which tells a reader the manifest is malformed. It is not:
// it is unbuilt, which is a different problem with a different fix, and only the mesh is in a
// position to tell them apart (novox/hq ADR 0073).
func built(resource map[string]any, module string) error {
if fmt.Sprint(resource["type"]) != "container" {
return nil
}
artifact, ok := resource["artifact"].(string)
if !ok || artifact == "" {
return nil
}
return fmt.Errorf(
"%s has not been built for this mesh: %v is its %q artifact, and no build has published "+
"one. Build it — `build <repository> --path <path>` — and the module will name what "+
"came out instead",
module, resource["id"], artifact)
}
// publishedOn puts the machine's own port on the outside of a container's mapping.
//
// **A module writes the port the software uses; the mesh says where the machine puts it**
// (novox/hq ADR 0038). So `"5432"` means *publish what the software calls 5432*, and this fills in
// the half only the mesh can know.
//
// A mapping written the long way — `"5433:5432"` — is left exactly as it is. Some things genuinely
// must be pinned down by hand, and quietly overruling somebody who wrote both halves would be
// worse than not offering the short form at all.
func publishedOn(resource map[string]any, module string, with mcat.Rendering) {
if fmt.Sprint(resource["type"]) != "container" {
return
}
listed, ok := resource["ports"].([]any)
if !ok {
return
}
out := make([]any, 0, len(listed))
for _, entry := range listed {
written := fmt.Sprint(entry)
if strings.Contains(written, ":") {
// Written the long way, and left alone — unless this node was given a machine port for
// it (novox/hq ADR 0100): the foundation's ports are the node's, and a manifest's
// number is only the default. The outer port only; an address and the software's
// port stay as written.
out = append(out, givenOuter(written, with.Given[module]))
continue
}
// A short form may carry the protocol — `"3478/udp"` — and the number is what the mesh
// assigns for; the protocol rides along. Read as one token, the `/udp` made the whole
// entry "not a port", and passing it through let the runtime publish it wherever it
// liked: unifi's STUN and discovery landed on random machine ports while every TCP pin
// beside them held.
mapping, protocol := written, ""
if cut := strings.LastIndex(written, "/"); cut >= 0 {
mapping, protocol = written[:cut], written[cut:]
}
wanted, err := strconv.Atoi(strings.TrimSpace(mapping))
if err != nil {
// Not a port at all. Passed through, so the host refuses it with its own words rather
// than this quietly dropping something somebody meant.
out = append(out, written)
continue
}
out = append(out, fmt.Sprintf("%d:%d%s", renderingMachinePort(with, module, wanted), wanted, protocol))
}
resource["ports"] = out
}
// givenOuter rewrites the machine side of a long-form mapping to the port this node was given for
// it, when it was given one.
//
// **Under either of the mapping's names.** A node gives a port by the number the module names it
// by, and a module publishing `"2222:22"` may say it listens on 22 or on 2222 — GivenPorts accepts
// both and answers to both, so looking the machine side up first and the container's port second
// finds the same number either way. Read only by the container's port, this moved nothing for the
// module that declares the machine side, and the setting was refused before it got here.
func givenOuter(written string, given map[int]int) string {
if len(given) == 0 {
return written
}
mapping, protocol := written, ""
if cut := strings.LastIndex(written, "/"); cut >= 0 {
mapping, protocol = written[:cut], written[cut:]
}
parts := strings.Split(mapping, ":")
if len(parts) < 2 {
return written
}
inner, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1]))
if err != nil {
return written
}
at, ok := given[inner]
if outer, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-2])); err == nil {
if machine, named := given[outer]; named {
at, ok = machine, true
}
}
if !ok {
return written
}
parts[len(parts)-2] = strconv.Itoa(at)
return strings.Join(parts, ":") + protocol
}
// ServedOn is what a provider tells a consumer, with the port that machine actually uses.
//
// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three
// times — for the rule set, for what a consumer is told, and for what the runtime publishes — and
// nothing checked that the three agreed. This is what fills the second in.
//
// The assignment wins when there is one, and the declared port stands in when there is not: a
// caller composing without a store still gets something coherent, and a mesh that has assigned one
// tells the truth about where it put it.
//
// Only when the module offers exactly one port. A module offering several has not said which
// belongs to which provision, and guessing would give a consumer a port that answers something
// else — so it keeps whatever the manifest said, which may be nothing.
func ServedOn(m mcat.Manifest, provision string, ports map[int]int) map[string]any {
serves := m.Serves[provision]
out := make(map[string]any, len(serves)+1)
for k, v := range serves {
out[k] = v
}
if _, said := out["port"]; said {
// Written by hand. Redirected to wherever the machine put it, and otherwise left alone.
if number, ok := asPort(out["port"]); ok {
if at, known := ports[number]; known {
out["port"] = at
}
}
return out
}
if len(m.Listens) != 1 {
return out
}
wanted := m.Listens[0].Port
if at, known := ports[wanted]; known {
out["port"] = at
} else {
out["port"] = wanted
}
return out
}
func asPort(v any) (int, bool) {
switch n := v.(type) {
case int:
return n, true
case float64:
return int(n), true
}
return 0, false
}
// atMachinePort redirects a `port` written by a module on this machine to where this machine
// actually published it (novox/hq 04-ISSUES/038, ADR 0066).
//
// **A module writes the port its software uses; only the mesh knows where the machine put it.** A
// bare `ports` mapping is assigned a host port when the declaration is composed, which is after
// everything a module wrote has been read — so any number carried out of a manifest is the declared
// one until it passes through here. Both directions need it: what a co-located consumer is TOLD
// about a provider, and what a co-located workload TELLS a provider about itself.
//
// Named by the module the port belongs to, which is not always the one being talked about: a route
// contribution's port is the contributing workload's, never the proxy's.
//
// A fresh map when it changes anything, so the manifest-derived value handed back by the resolver —
// or by a grant gathered elsewhere — is never mutated under a caller that still holds it. Keyed by
// the *declared* port, so applying it a second time is a no-op: once redirected the value is the
// machine port, which is not itself a key.
func atMachinePort(serves map[string]any, module string, ports map[string]map[int]int) map[string]any {
number, ok := asPort(serves["port"])
if !ok {
return serves
}
at, known := ports[module][number]
if !known || at == number {
return serves
}
copied := make(map[string]any, len(serves))
for k, v := range serves {
copied[k] = v
}
copied["port"] = at
return copied
}
// AtPublishedPort redirects a contribution's port to where the CONSUMER's own machine published it.
//
// The same redirection as [atMachinePort], for the one caller that cannot reach it. A grant is
// assembled on the providing machine out of a consumer that lives on another one, so that
// consumer's assignments are fetched there and handed in, rather than looked for here where they
// are not. Without it a proxy told to reach a workload on another machine is told the port the
// workload's *software* uses, and dials a number that machine never published — the same fault as
// novox/hq 04-ISSUES/038, one node over.
func AtPublishedPort(values map[string]any, module string, published map[int]int) map[string]any {
return atMachinePort(values, module, map[string]map[int]int{module: published})
}
// preparationTarget is the resource a module's preparation runs before: its own workload.
//
// The first container carrying an artifact this module built, and not itself a step — that is the
// thing that runs the module's code, and therefore the thing whose state must be ready. Empty when
// the module prepares nothing, or when nothing it declares could run its code.
//
// **A module with two own workloads gates the first of them.** Five modules in the catalogue declare
// more than one container of their own, none of them preparing anything today. If one ever does and
// its second workload shares the state, the gate is in front of the first — stated here because the
// alternative is a field asking an author to restate what the mesh can see.
func preparationTarget(m mcat.Manifest) string {
if !m.Prepares {
return ""
}
for _, r := range m.Resources {
// A container, or a process the host runs from a bundle the module built (novox/hq issue
// 213): the same program in the same context, hosted as a unit rather than a container.
kind := fmt.Sprint(r["type"])
if (kind != "container" && kind != "process") || !ownArtifact(r, m.Module) {
continue
}
if once, _ := r["run-once"].(bool); once {
continue
}
if r["schedule"] != nil {
continue
}
return fmt.Sprint(r["id"])
}
return ""
}
// ownArtifact is whether a resource runs something this module built, in either spelling a manifest
// may be in: naming the artifact, before a build resolved it, or carrying the reference a build
// recorded — this mesh's own store, under this module's name.
func ownArtifact(resource map[string]any, module string) bool {
if named, _ := resource["artifact"].(string); named != "" {
return true
}
image, _ := resource["image"].(string)
// A process or an archive carries what was built as its source (novox/hq issue 213).
source, _ := resource["source"].(string)
return strings.HasPrefix(image, mcat.ArtifactStoreScheme+module+"/") ||
strings.HasPrefix(source, mcat.ArtifactStoreScheme+module+"/")
}
// prepared is the module's own resource as the step that prepares its state: the same image, the same
// context, run to completion with the mesh's preparation argument.
//
// Three things are taken away rather than copied, each because the step runs while the version it
// prepares for is still running. A published port cannot be bound twice, and a step that tried would
// fail for a reason that has nothing to do with the state. A fixed address cannot be held twice, for
// the same reason. And a cadence is what a step is the opposite of: a container runs once and gates,
// or on a schedule, or stays up, never two (ADR 0053).
func prepared(from map[string]any) map[string]any {
step := map[string]any{}
for k, v := range from {
step[k] = v
}
// **A hyphen, not a dot.** A resource's id is `<module>.<its own id>`, and a module's name may
// itself contain a dot (`novox.be`), so the module is everything before the *last* dot — which
// only works if what the mesh derives adds no dot of its own.
step["id"] = fmt.Sprint(from["id"]) + "-prepare"
step["name"] = fmt.Sprint(from["name"]) + "-prepare"
step["run-once"] = true
if fmt.Sprint(from["type"]) == "process" {
// A process says its whole command: the program, then its arguments. The step is the same
// program asked to prepare (novox/hq issue 213). It replaces nothing — what the process
// replaces is handed over to the process, never to the step that runs before it — and a
// step is not restarted, it runs again when what it reads changed, which `restart-on` says.
run := stringsIn(from["run"])
if len(run) > 0 {
step["run"] = []any{run[0], mcat.PreparationArgument}
}
delete(step, "replaces")
} else {
step["args"] = []any{mcat.PreparationArgument}
}
delete(step, "ports")
delete(step, "ip")
delete(step, "schedule")
delete(step, "reload-on")
return step
}
// endpointPortOf is the port the endpoint a route serves listens on: looked up by the name the route
// gives, or read from the port it repeats (novox/hq ADR 0138).
//
// `ports` maps this module's endpoint names to their ports, computed once per module rather than
// re-scanned per contribution.
func endpointPortOf(values map[string]any, ports map[string]int) (int, bool) {
if name, ok := values[mcat.RouteEndpoint].(string); ok {
if port, found := ports[strings.TrimSpace(name)]; found {
return port, true
}
}
return asPort(values["port"])
}
// endpointPorts is a module's endpoint names against the ports they listen on.
func endpointPorts(m mcat.Manifest) map[string]int {
out := map[string]int{}
for _, l := range m.Listens {
if name := strings.TrimSpace(l.Name); name != "" {
out[name] = l.Port
}
}
return out
}
// portOfEndpoint fills in the port of the endpoint a contribution names, in place.
//
// **A contribution that names an endpoint must still carry that endpoint's port**, because everything
// downstream reads the port: the provider is told where to reach the consumer, and the machine-side
// redirection that turns a declared port into the number the machine published is keyed on it
// (atMachinePort). A route that named only its endpoint left the proxy with no port at all, and a
// proxy with no port has nothing to dial.
//
// Found before it shipped and after the catalogue had already been changed to name endpoints — the
// manifests were merged and the mesh had not yet picked them up, so nothing was broken yet. The
// declared port, not the machine one: the redirection happens later and is keyed on the declared
// number, so filling in the machine port here would be redirected a second time or not at all.
func portOfEndpoint(values map[string]any, ports map[string]int) {
if values == nil {
return
}
if _, already := values["port"]; already {
// A route that says both is its own answer; the older shape repeated the port and is still read.
return
}
name, ok := values[mcat.RouteEndpoint].(string)
if !ok {
return
}
if port, found := ports[strings.TrimSpace(name)]; found {
values["port"] = port
}
}
// secretsReadBy is the file resources of this module's own secrets that a container or a daemon reads
// — named in its volumes, its environment or its env-files by the secret's placed path — as
// restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which
// the host refuses a restart-on for (it runs again anyway, and reads the file afresh).
//
// **A member of a secret family given here is read the same way** (novox/hq issue 405, ADR 0283 decision 6):
// it is an own secret placed at its own path, and a container that mounted it would keep the value it
// started with when the operator gives it again. given is the module's own secrets sealed to this
// machine; a member not given is no file, so nothing restarts on it.
func secretsReadBy(resource map[string]any, m mcat.Manifest, given map[string]string) []string {
kind := fmt.Sprint(resource["type"])
if kind != "container" && kind != "process" {
return nil
}
if resource["schedule"] != nil || resource["run-once"] == true {
return nil
}
var mentioned []string
for _, key := range []string{"volumes", "env", "env-file"} {
mentioned = append(mentioned, stringsIn(resource[key])...)
}
paths := map[string]string{}
for name, own := range m.OwnSecrets.Plain() {
if own.IssuedBy == mcat.IssuedOutside && given[name] == "" {
continue // not given, so not placed: nothing to restart on (novox/hq issue 401)
}
paths[name] = own.Path
}
for name, sealed := range given {
if own, family, ok := m.OwnSecrets.Lookup(name); ok && family != "" && sealed != "" {
paths[name] = own.Path
}
}
var out []string
for _, name := range sortedKeys(paths) {
path := paths[name]
if path == "" {
continue
}
for _, s := range mentioned {
// A volume is `source:destination[:mode]`; an env value or an env-file is the path itself.
if s == path || strings.HasPrefix(s, path+":") {
out = append(out, m.Module+"."+mcat.NeedID(name))
break
}
}
}
return out
}
// stringsIn is every string in a list or a map's values; nothing for anything else.
func stringsIn(v any) []string {
switch x := v.(type) {
case []any:
var out []string
for _, item := range x {
if s, ok := item.(string); ok {
out = append(out, s)
}
}
return out
case []string:
return x
case map[string]any:
var out []string
for _, k := range sortedKeys(x) {
if s, ok := x[k].(string); ok {
out = append(out, s)
}
}
return out
case map[string]string:
var out []string
for _, k := range sortedKeys(x) {
out = append(out, x[k])
}
return out
}
return nil
}
// withRestartOn is a resource's restart-on list with these ids added once each.
func withRestartOn(have any, add []string) []any {
var out []any
seen := map[string]bool{}
for _, id := range reflectsRenamed("", have) {
s := fmt.Sprint(id)
if !seen[s] {
seen[s] = true
out = append(out, s)
}
}
for _, id := range add {
if !seen[id] {
seen[id] = true
out = append(out, id)
}
}
return out
}
// accountsFirst splits a module's resources into its accounts and everything else, each in the order
// written.
//
// **An account declared only to be put in groups keeps its written place** (novox/hq ADR 0252). It is not
// the module's own account — that one says how it logs in or where it lives — but one the module needs in
// a group: the operator's, in the group of a daemon the module installs. The group is made by the package
// that installs the daemon, so put first, the account was put in a group the machine did not have yet, and
// only the next apply healed it. Written after its packages, it is applied after them.
func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any) {
for _, r := range resources {
if fmt.Sprint(r["type"]) == "user" && !onlyGroups(r) {
accounts = append(accounts, r)
continue
}
rest = append(rest, r)
}
return accounts, rest
}
// onlyGroups is a user resource that names groups and nothing else of the account: no shell, home or
// lingering.
func onlyGroups(r map[string]any) bool {
for _, field := range []string{"shell", "home", "linger"} {
if v, ok := r[field]; ok && v != nil && v != "" {
return false
}
}
return len(stringsIn(r["groups"])) > 0
}
// generatedHere is what each computed module's generator answers for this machine, by module —
// absent for a module whose machine is not yet part of what it generates — held to the rule every
// module is held to: no two modules on a machine declare one path, unit or name, nor one package present
// and absent (novox/hq issue 190, step 5; ADR 0222, ADR 0303).
//
// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
// resources it will declare — they exist only once its generator has answered for this machine,
// here — so a generated resource writing into another module's file was never seen. That is how
// the private network came to declare the container runtime's daemon file and service beside the
// runtime's own module. Asked once, so what is checked is exactly what is declared.
func resolutionGeneratedHere(r mcat.Resolution, with mcat.Rendering) (map[string][]map[string]any, error) {
generated := map[string][]map[string]any{}
placed := make([]mcat.Manifest, 0, len(r.Modules))
for _, m := range r.Modules {
if m.Computed == "" {
placed = append(placed, m)
continue
}
generator, known := with.Generators[m.Computed]
if !known {
return nil, fmt.Errorf(
"%s says its resources are computed by %q, and this control plane has no %q",
m.Module, m.Computed, m.Computed)
}
resources, part, err := generator.Resources(r.Node)
if err != nil {
return nil, err
}
computed := m
computed.Resources = nil
if part {
generated[m.Module] = resources
computed.Resources = resources
}
placed = append(placed, computed)
}
if len(generated) == 0 {
return generated, nil // nothing resolution has not already judged
}
if problems := checkResources(placed); len(problems) > 0 {
return nil, fmt.Errorf("what the mesh computes for this machine collides with a module's own: %s",
strings.Join(problems, "; "))
}
return generated, nil
}
Reference in New Issue View Git Blame Copy Permalink