novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

stage 5 of 8 for issue 1389

Browse Source
This commit is contained in:
mesh-tools
2026-10-20 11:15:01 +00:00
parent 15f0888b4f
commit 66daf0cc27
1 changed files with 0 additions and 575 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/plan.go
-575
View File
@@ -153,499 +153,6 @@ func reportLeftOut(node string, declared sendable) {
}
}
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
//
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
var user string
switch name {
case "broker":
user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
case broker.ProvisionerSecret:
// And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11,
// design 57 P3): the same refusal, for the same reason.
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
default:
return nil
}
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return err
} else if !minted {
return fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
module, node, user, module, node)
}
return nil
}
// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside
// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again.
func givenUnderAnOldKey(module, node, name string) error {
return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
"since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node)
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
inv := open.inventory
grants, withheld, unbound, err := grantsFor(ctx, open, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
//
// **Assigned here rather than written by a module**, because a module is written once and
// assigned anywhere: any number it picks is a guess about a machine it has never seen. Made
// before the declaration is composed, because the container's mapping, the rule set and what a
// consumer is told are all derived from it.
// What this machine was already given, for a composition that may not allocate.
already := map[string]map[int]int{}
if choosing != Allocating {
held, err := inv.PortsFor(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
for _, a := range held {
if already[a.Module] == nil {
already[a.Module] = map[int]int{}
}
already[a.Module][a.Wanted] = a.Machine
}
}
// The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's
// ports, as genesis chose them. A given port wins over anything assigned and over a manifest's
// own long-form mapping.
given := map[string]map[int]int{}
for _, m := range plan.Modules {
g, err := catalogue.GivenPorts(m, settings[m.Module])
if err != nil {
// A given port its definition no longer publishes: the module is left out of the
// declaration, by name, when it is composed (novox/hq ADR 0163, rule 6) — never the
// machine refused here for it.
continue
}
if g != nil {
given[m.Module] = g
}
}
// And one holder per machine port across the node's modules: settings written before this was
// refused where they are set are refused here rather than composed into two containers on
// one port.
holders := map[int]string{}
for _, m := range plan.Modules {
for _, at := range given[m.Module] {
if other, twice := holders[at]; twice && other != m.Module {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+
"both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module,
at, node)
}
holders[at] = m.Module
}
}
ports := map[string]map[int]int{}
for _, m := range plan.Modules {
for _, l := range m.Listens {
if at, isGiven := given[m.Module][l.Port]; isGiven {
if ports[m.Module] == nil {
ports[m.Module] = map[int]int{}
}
ports[m.Module][l.Port] = at
continue
}
// **Only a port the module actually publishes is the mesh's to move.** A container's
// mapping is the thing that translates; without one the software binds what it binds,
// and an assignment would not move the service — it would open the wrong number in the
// rule set and leave the real one shut. Recorded either way, because this map means
// *where this module's port is on this machine* and every reader of it needs that
// answer whether or not the mesh was the one who chose it.
where, mayAssign := m.MachineSide(l.Port)
switch {
case mayAssign && choosing == Allocating:
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s needs %d reachable on %s and it could not be assigned: %w",
m.Module, l.Port, node, err)
}
where = at.Machine
case mayAssign:
// Whatever was chosen last time, and nothing if there was no last time.
if at, known := already[m.Module][l.Port]; known {
where = at
}
}
if ports[m.Module] == nil {
ports[m.Module] = map[int]int{}
}
ports[m.Module][l.Port] = where
}
}
// And each module's own secrets — a superuser password, an administrator, an account. Made
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
foreseen := map[string]map[string]bool{}
// **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11,
// design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for
// this machine. Every other module's are made and sealed as before.
hands, err := engineTakesHanded(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)}
for _, m := range plan.Modules {
if handing.Handing(m) {
continue
}
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
// placed, and one not given is nothing — the module says it waits for it.
for _, family := range m.OwnSecrets.Families() {
members, err := inv.GivenMembers(ctx, node, m.Module, family)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
for _, g := range members {
if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family {
continue // a longer family's member, or a name no longer of this family
}
if !g.Current {
if choosing == Allocating {
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name)
}
continue
}
if needed[m.Module] == nil {
needed[m.Module] = map[string]string{}
}
needed[m.Module][g.Name] = g.Sealed
}
}
for name, own := range m.OwnSecrets.Plain() {
// **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for
// a family): only a value a person gave is placed. One not given is nothing — no file, and an empty
// value where a file of the module names it — so the module can tell it was not given, where a value
// the mesh made looked given and wrong. A value the mesh made for it before is not one either.
if own.IssuedBy == catalogue.IssuedOutside {
g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if !given {
continue
}
if !g.Current {
if choosing == Allocating {
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name)
}
continue
}
if needed[m.Module] == nil {
needed[m.Module] = map[string]string{}
}
needed[m.Module][name] = g.Sealed
continue
}
// Minted on the send path and only read on every other. Making one is an insert, and
// a question that writes is a question that can block against the machine it is about.
var sealed string
var err error
if choosing == Allocating {
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
if err == nil && !held && choosing == Foreseeing {
// Asked ahead of the send: what the send would do about it, by the send's own
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
// words (novox/hq issue 275).
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if foreseen[m.Module] == nil {
foreseen[m.Module] = map[string]bool{}
}
foreseen[m.Module][name] = true
continue
}
if err == nil && !held {
// Never issued, so this machine cannot be running it. Left out rather than
// invented: an empty string here would compose a declaration that differs
// from what would be sent, and the comparison this feeds would then be
// answering about a declaration nothing will ever push.
continue
}
}
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if needed[m.Module] == nil {
needed[m.Module] = map[string]string{}
}
needed[m.Module][name] = sealed
}
}
// **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that
// opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that
// contract nothing here is asked and the composition is as it was.
var core coreBoxes
opensBox, err := engineOpensVaultBox(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if opensBox {
core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool {
return handing.Handing(m)
}, choosing)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
}
// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
// rather than stored: the node's key does not change, so signing again produces an equally
// valid certificate and there is nothing to keep in step.
var certificate, authority string
for _, m := range plan.Modules {
if m.Certificate == nil {
continue
}
issued, meshCA, err := certificateFor(ctx, open, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
certificate, authority = issued, meshCA
break
}
// And who else is on the private network, which is what a rule saying "from the mesh"
// resolves to. Every node's address, including this one's: a machine reaching itself by its
// own overlay address rather than by loopback is ordinary, and leaving it out would filter
// the node's own traffic to itself with no rule naming why.
// One reading of the catalogue for the three questions below that resolve the whole mesh.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
private, err := onThePrivateNetwork(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The private network's range, offered to a module as ${machine:mesh-range} — a module that must
// name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here
// rather than hardcoding a value it cannot know.
meshRange, err := overlayRange(ctx, inv)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The artifact store as this node reaches it now — the address every image and archive the
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
// built, so a reference recorded with an address before that is re-routed too.
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
held, err := inv.Held(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
built := make(map[string]bool, len(held))
for repository := range held {
built[repository] = true
}
// And every machine's name, so a container can reach one. The same set that writes the
// machine's own hosts file — one reading, so a container and its machine cannot disagree
// about where another machine is.
names, err := namesInTheMesh(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Each machine's operator account, so an ssh Host block can name the login for every node
// (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to.
allNodes, err := inv.Nodes(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
accounts := map[string]string{}
for _, n := range allNodes {
if n.Account != "" {
accounts[n.Name] = n.Account
}
}
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
// answer for any of them. The roster once carried every routed name, public ones included, and a
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
machines := make(map[string]string, len(names))
for name, at := range names {
machines[name] = at
}
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
// to forward.
zones, err := zonesInTheMesh(ctx, open)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
// lists every holder of the mesh's resolver.
replicas, err := replicatedHolders(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
// Nothing the mesh itself needs is opened beyond what a module declares.
var foundation []int
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
// export changes whenever any secret in the mesh is made or rotated, so the vault's declaration
// changes with it and the vault node is sent again: that is what keeps its copy current, and
// the cost is one two-table read per composition of the vault's node, in every mode.
var kept *catalogue.KeptExport
for _, m := range plan.Modules {
if m.Keeps == "" {
continue
}
if kept, err = inv.OperatorExport(ctx); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
break
}
// Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and
// the declaration carries openings and the mesh's guard in place of a filter.
record, err := inv.NodeByName(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// And, on an adopted node, which modules were taken there: the guard is derived from those
// only (novox/hq ADR 0103).
var taken map[string]bool
if record.Adopted {
list, err := inv.Taken(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
taken = map[string]bool{}
for _, m := range list {
taken[m] = true
}
}
// Where this node put the foundation's servers, for the control plane's own connections
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The bus's user list, for the machine that runs the bus. Composed per push rather than kept,
// because it is a function of the mesh's records and a kept copy could disagree with them.
busUsers, err := composeBusUsers(ctx, inv, plan.Modules)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
memberships, err := inv.BusMemberships(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Which of this machine's links face outside, which is what the derived filter is written
// around (novox/hq ADR 0140). Reported by the machine, never set.
outwardLinks, err := inv.OutwardLinksOf(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// And which of the networks it is attached to count as its local network (novox/hq ADR 0340):
// private ranges only, on a link facing outside it. Reported by the machine, never set.
local, err := localNetworksOf(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
// 0222): the artifact store as this network reaches it, which the container runtime is told to
// trust (ADR 0082). The same address composed into every reference the mesh built.
var reach map[string]string
if artifactStore != "" {
reach = map[string]string{"mesh-artifact-store": artifactStore}
}
readsHealth, err := engineReadsHealth(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
judgesRoot, err := engineJudgesRoot(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
readsBusy, err := engineReadsBusy(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
ReadsHealth: readsHealth,
JudgesRoot: judgesRoot,
ReadsBusy: readsBusy,
Hands: hands,
Provisioner: provisionerAccount(plan),
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
VaultSealed: core.sealed, NotPlaced: core.notPlaced,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
LocalNetworks: catalogue.CountedLocalNetworks(local),
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
}, record, nil
}
// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240
// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the
// field, because it parses strictly and would refuse the whole declaration for it.
func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.ReadinessContract, nil
}
// engineJudgesRoot says whether a machine's node-engine judges a user's declared `root` (novox/hq ADR 0266),
// by its own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.RootContract, nil
}
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
@@ -656,23 +163,6 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string)
return had && stated.Contract >= link.BusyContract, nil
}
// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone
// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the
// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before.
func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return takesHanded(stated, had), nil
}
// takesHanded is whether a machine's newest statement (had: it ever stated one) is of an engine that writes a
// filled file on the provisioner client's word alone.
func takesHanded(stated inventory.NodeHealth, had bool) bool {
return had && stated.Contract >= link.HandedContract
}
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
// the process of the module running it runs as, empty where no such module is on the machine.
func provisionerAccount(plan catalogue.Resolution) string {
@@ -710,71 +200,6 @@ func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
return out
}
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
// 0199): the zone settled from that node's settings, the node's private address, the port the
// answering listen is published on there.
//
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
// suffix or a node's public domain — is refused here, by name.
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
}
address := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
address[p.Name] = p.Address
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
}
var zones []catalogue.ZoneAt
var public []string
for _, n := range nodes {
plan, _, err := resolveNode(ctx, open, n.Name)
switch {
case unresolvable(err):
continue
case err != nil:
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
}
if plan.PublicDomain != "" {
public = append(public, plan.PublicDomain)
}
for _, m := range plan.Modules {
if m.Zone == nil {
continue
}
at := address[n.Name]
if at == "" {
// Not on the private network yet: nothing could reach its answerer.
continue
}
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
if err != nil {
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
}
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
if err != nil {
return nil, err
}
zones = append(zones, *z)
}
}
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
}
return zones, nil
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows