jochen
e2622fd031
assign and unassign say only what they changed on their node; push <node> lists that node's, push to many counts each and points at status. The once-per-change log is the serving controller's alone: a one-shot command starts with no memory, so it logged every node on every call.
1418 lines
56 KiB
Go
1418 lines
56 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/licences"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// working out what one machine should be.
|
|
//
|
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
|
|
|
// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
|
|
// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
|
|
// unreachable, a key could not be read — and says nothing about the node at all.
|
|
//
|
|
// The distinction exists because three callers gather something across every machine and must carry
|
|
// on when one machine's set is broken. Each of them read a plain error as "their set does not
|
|
// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
|
|
// roster of routed names that is not a degraded answer but a false one: it states, to every machine
|
|
// at once, that another machine's names do not exist. A control node spent hours replacing every
|
|
// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
|
|
// the read failed, one name left the roster, and the roster is part of every container's identity
|
|
// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
|
|
//
|
|
// So: skip a node that cannot resolve, and never a node that could not be read.
|
|
type notResolvable struct{ err error }
|
|
|
|
func (n notResolvable) Error() string { return n.err.Error() }
|
|
func (n notResolvable) Unwrap() error { return n.err }
|
|
|
|
// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
|
|
// being unable to answer.
|
|
func unresolvable(err error) bool {
|
|
var n notResolvable
|
|
return errors.As(err, &n)
|
|
}
|
|
|
|
// planFor works out everything a node should run, from what was assigned to it.
|
|
//
|
|
// A failure to compose the node's own modules is wrapped as notResolvable; every other failure is
|
|
// returned as it is. Callers gathering across the mesh must tell them apart — see notResolvable.
|
|
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
|
|
inv := open.inventory
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
assigned, err := inv.Assigned(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
capabilities, err := inv.ProfileOf(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
var site string
|
|
for _, p := range places {
|
|
if p.Name == nodeName {
|
|
site = p.Site
|
|
}
|
|
}
|
|
|
|
world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
world.Pinned, err = inv.PinsFor(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
// What this mesh can answer with a record rather than a machine, and which record each of
|
|
// this node's modules was put on. Read across a context boundary by name, which is what
|
|
// crossing one is allowed to carry (novox/hq ADR 0008).
|
|
world.Licences, world.Using, err = licencesFor(ctx, open, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
// The domain this node composes its routed names under (novox/hq ADR 0066). A route
|
|
// contribution carries only a label; the resolver joins <label>.<public-domain> for this node,
|
|
// so the fact travels on the node it belongs to rather than being looked up where the name is
|
|
// composed.
|
|
publicDomain, err := inv.PublicDomainOf(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
// The operator account this node logs a person in as, and where its home is (novox/hq to-be
|
|
// 29) — carried so a home-scoped file's owner and path resolve for this machine.
|
|
who, err := inv.NodeByName(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
resolved, err := catalogue.Resolve(shelf, assigned,
|
|
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
|
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
|
Account: who.Account, AccountHome: who.AccountHome}, world)
|
|
if err != nil {
|
|
// The node's own set does not compose. Marked, because this is the only failure here that
|
|
// a mesh-wide gatherer may pass over — see notResolvable.
|
|
return catalogue.Resolution{}, nil, notResolvable{err}
|
|
}
|
|
logUnheld(nodeName, resolved.Unheld)
|
|
|
|
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
|
// password a provider is told to create is the one its consumer was given — and sealed to
|
|
// this node before it was ever written down, so nothing between here and there can read it.
|
|
for i, n := range resolved.Needs {
|
|
if n.ByRecord {
|
|
// Answered by something the mesh holds, so there is no pair-wise secret between two
|
|
// machines. Its key was supplied by a person and sealed to this node then; the mesh
|
|
// discarded the plaintext and cannot make another.
|
|
sealed, err := keyFor(ctx, open, n.From, nodeName, n.For)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
resolved.Needs[i].Sealed = sealed
|
|
// If this holder is the licence's manager, hand it the manager node's PUBLIC sealing key
|
|
// in its bound facts (novox/hq ADR 0050). It is safe to disclose — a public key — and it
|
|
// is what the manager module needs to re-seal a rotated refresh token to this same node,
|
|
// having been given no private key of its own. A consumer holder gets none.
|
|
pub, err := managerPublicKeyFor(ctx, open, inv, n.From, nodeName, n.For)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
if pub != "" {
|
|
serves := map[string]any{}
|
|
for k, v := range resolved.Needs[i].Serves {
|
|
serves[k] = v
|
|
}
|
|
serves["manager_public_key"] = pub
|
|
resolved.Needs[i].Serves = serves
|
|
// The manager holder: its empty pre-adoption refresh token is a waiting state, not a
|
|
// missing consumer key, so the declaration tolerates it rather than refusing.
|
|
resolved.Needs[i].Manager = true
|
|
}
|
|
continue
|
|
}
|
|
var secret inventory.Secret
|
|
var err error
|
|
if n.SharedOwn != "" {
|
|
// The provider's one credential, sealed to this consumer too (novox/hq ADR 0158).
|
|
secret, err = inv.SharedSecretFor(ctx, n.Name, nodeName, n.For, n.From, providerModuleOf(resolved, open, ctx, n), n.Local, n.SharedOwn)
|
|
} else {
|
|
secret, err = inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
|
|
}
|
|
if err != nil {
|
|
// Said rather than skipped. A machine that resolves cleanly and receives no
|
|
// credential is one that will fail to authenticate at some later, less obvious
|
|
// moment.
|
|
return catalogue.Resolution{}, nil, fmt.Errorf(
|
|
"%s on %s needs %s from %s and no credential could be made for it: %w",
|
|
n.For, nodeName, n.Name, n.From, err)
|
|
}
|
|
resolved.Needs[i].Sealed = secret.ForConsumer
|
|
}
|
|
|
|
// Settings for everything that resolved, including modules nobody assigned directly: a
|
|
// requirement pulled in by something else is still configurable, and finding out that it is
|
|
// not only when you try would be an arbitrary line nobody could predict.
|
|
//
|
|
// A setting that reaches nothing, or cannot compose with the definition it was stored for,
|
|
// no longer refuses the machine here: it is judged where it is stored, and a definition that
|
|
// moved under it costs that module its place in the declaration, said by name (novox/hq ADR
|
|
// 0163, rule 6 — see Compose).
|
|
settings := catalogue.SettingsBy{}
|
|
for _, m := range resolved.Modules {
|
|
layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
if len(layers) == 0 {
|
|
continue
|
|
}
|
|
settings[m.Module] = layers
|
|
}
|
|
return resolved, settings, nil
|
|
}
|
|
|
|
// theRestOfTheMesh is what every other node holds and offers.
|
|
//
|
|
// Two things at once because they come from the same place — resolving the other nodes — and
|
|
// because both are facts about what is actually running rather than records that could disagree
|
|
// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node
|
|
// runs; neither is a table somebody keeps up to date.
|
|
//
|
|
// **Two passes over the others.** What a node offers the mesh needs that node resolved, and
|
|
// resolving it may need what the mesh offers. So the first pass takes brokered requirements on
|
|
// trust and answers only *what does each node offer*; the second answers everything with that in
|
|
// hand. Nothing is ever declared from the first.
|
|
func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|
shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
|
|
|
// Every node, not only the placed ones. A machine that was never put on the private network
|
|
// still runs modules, still holds claims, and still offers whatever it offers.
|
|
// **Who holds each seat on record, before anything is resolved** (novox/hq ADR 0131). Both
|
|
// passes below need it: without it, the assignment standing beside a seat's holder — the next
|
|
// holder, waiting for the handover — is refused as a second holder, and its node's whole set
|
|
// with it.
|
|
holdings, err := inv.Holdings(ctx)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
siteOf := map[string]string{}
|
|
for _, p := range places {
|
|
siteOf[p.Name] = p.Site
|
|
}
|
|
// Which machines are actually on the private network, and what they are called there. Not
|
|
// "has an address" — that was true of every placed machine and told you nothing about whether
|
|
// anything could reach it. It is what resolved the module.
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
|
|
type candidate struct {
|
|
node catalogue.Node
|
|
assigned []string
|
|
}
|
|
var others []candidate
|
|
for _, n := range nodes {
|
|
if n.Name == exclude {
|
|
continue
|
|
}
|
|
theirs, err := inv.Assigned(ctx, n.Name)
|
|
if err != nil || len(theirs) == 0 {
|
|
continue
|
|
}
|
|
caps, _ := inv.ProfileOf(ctx, n.Name)
|
|
others = append(others, candidate{
|
|
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
|
|
At: onNetwork[n.Name]}, theirs})
|
|
}
|
|
|
|
offered := map[string][]catalogue.Provider{}
|
|
var firstHeld []catalogue.Held
|
|
for _, o := range others {
|
|
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings})
|
|
if err != nil {
|
|
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
|
|
// as every other machine's plan can tell (novox/hq issue 188).
|
|
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
|
continue
|
|
}
|
|
firstHeld = append(firstHeld, got.Claims...)
|
|
for _, m := range got.Modules {
|
|
for _, name := range m.OffersAt(catalogue.ScopeMesh) {
|
|
// What that module says a consumer needs to know, with that node's settings on
|
|
// it: a port somebody moved on the provider is a port its consumers must be told
|
|
// about, and the two coming from different places is how they come to disagree.
|
|
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
offered[name] = append(offered[name], catalogue.Provider{
|
|
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
|
|
}
|
|
}
|
|
}
|
|
for k := range offered {
|
|
sort.Slice(offered[k], func(i, j int) bool {
|
|
return offered[k][i].Node < offered[k][j].Node
|
|
})
|
|
}
|
|
|
|
// **The second pass is given the first pass's holdings.** A seat's holder answers a requirement
|
|
// with several providers (novox/hq ADR 0110), so a node consuming one resolves only once the
|
|
// holder is known. Without them its set is refused here, and a refused node's own claims drop
|
|
// out of what the mesh holds — so a second holder of one of its seats would pass unrefused.
|
|
world := catalogue.World{Offered: offered, Held: firstHeld, Holdings: holdings}
|
|
var held []catalogue.Held
|
|
for _, o := range others {
|
|
// Each machine is resolved with its own pins, as its plan is: a machine that needs one to
|
|
// settle two providers would otherwise be refused here and vanish from the mesh — every
|
|
// seat it holds unheld, every build that needs one refused (2026-10-01, the control node;
|
|
// novox/hq issue 188).
|
|
theirs := world
|
|
if pins, err := inv.PinsFor(ctx, o.node.Name); err == nil {
|
|
theirs.Pinned = pins
|
|
}
|
|
got, err := catalogue.Resolve(shelf, o.assigned, o.node, theirs)
|
|
if err != nil {
|
|
// Said only for the whole-mesh view. With one machine excluded, the others are
|
|
// resolved without its offers, and one that consumes them cannot resolve here by
|
|
// design — that is not the machine being dropped, it is the view being partial.
|
|
if exclude == "" {
|
|
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
|
}
|
|
continue
|
|
}
|
|
held = append(held, got.Claims...)
|
|
}
|
|
world.Held = held
|
|
return world, nil
|
|
}
|
|
|
|
// declarationFor is everything a node would be sent.
|
|
//
|
|
// One place, because there were three and one of them was written before credentials existed and
|
|
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
|
// `plan --json` handed to anything reading it.
|
|
func declarationFor(ctx context.Context, open *stores, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy) (sendable, error) {
|
|
gens, err := generators(ctx, open)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
// **Allocating, because `plan` is the send without the sending.** It is one machine, named by
|
|
// a person, who is asking what a push would do — so the port it shows and the secret it seals
|
|
// have to be the ones a push would use, and both are kept once chosen. Showing numbers that a
|
|
// later push would replace would make the command answer a question nobody asked.
|
|
//
|
|
// The line is not "a question may not write". It is who is asking and how often: this is a
|
|
// person, about one machine, on purpose. What may not write is the comparison the mesh runs
|
|
// over every machine to answer whether each is up to date — see Choosing.
|
|
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
|
}
|
|
|
|
// declarationWith is the same, for a caller that has already worked out the generators once and
|
|
// is about to use them for every node.
|
|
// Choosing says whether this composition may allocate what has not been allocated yet.
|
|
//
|
|
// **The comparison the mesh runs over every machine must not change what it is comparing.**
|
|
// Composing a declaration assigns each module a machine port and seals its secrets, and `status`
|
|
// composes one for every node to answer *is this machine running what I would send it* — so that
|
|
// question allocated, minted, wrote, and contended with the very machine it was asking about. A
|
|
// status polled every two seconds while a node applies is then two writers on the same rows,
|
|
// which is how it came to hang rather than answer.
|
|
//
|
|
// `plan` sits on the other side of this and allocates, because it is a person asking what a push
|
|
// would do to one named machine. The distinction is not question versus command; it is a person
|
|
// asking once about one machine versus the mesh asking continuously about all of them.
|
|
//
|
|
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
|
|
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
|
|
// machine "waiting" means — the read needs no number to be right about that.
|
|
type Choosing bool
|
|
|
|
const (
|
|
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
|
Allocating Choosing = true
|
|
// Reading is every question: what is assigned is used, and nothing is created.
|
|
Reading Choosing = false
|
|
)
|
|
|
|
func declarationWith(ctx context.Context, open *stores, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
|
gens map[string]catalogue.Generator, choosing Choosing) (sendable, error) {
|
|
with, record, err := renderingFor(ctx, open, node, plan, settings, gens, choosing)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
composed, err := plan.Compose(with)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
// And what was taken on it, said in every declaration it is sent from this one place.
|
|
adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
return sendable{Resources: composed.Resources, Adoption: adoption,
|
|
Received: composed.Received, Mesh: with.Mesh,
|
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
|
}
|
|
|
|
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
|
// for a reordering nobody made.
|
|
func sortedKeysOf(m map[string]string) []string {
|
|
if len(m) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
|
|
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
|
|
func reportLeftOut(node string, declared sendable) {
|
|
for _, m := range declared.LeftOut {
|
|
fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+
|
|
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
|
node, m, declared.leftOutWhy[m])
|
|
}
|
|
}
|
|
|
|
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
|
func renderingFor(ctx context.Context, open *stores, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
|
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
|
inv := open.inventory
|
|
grants, err := grantsFor(ctx, open, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
|
|
//
|
|
// **Assigned here rather than written by a module**, because a module is written once and
|
|
// assigned anywhere: any number it picks is a guess about a machine it has never seen. Made
|
|
// before the declaration is composed, because the container's mapping, the rule set and what a
|
|
// consumer is told are all derived from it.
|
|
// What this machine was already given, for a composition that may not allocate.
|
|
already := map[string]map[int]int{}
|
|
if choosing == Reading {
|
|
held, err := inv.PortsFor(ctx, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
for _, a := range held {
|
|
if already[a.Module] == nil {
|
|
already[a.Module] = map[int]int{}
|
|
}
|
|
already[a.Module][a.Wanted] = a.Machine
|
|
}
|
|
}
|
|
|
|
// The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's
|
|
// ports, as genesis chose them. A given port wins over anything assigned and over a manifest's
|
|
// own long-form mapping.
|
|
given := map[string]map[int]int{}
|
|
for _, m := range plan.Modules {
|
|
g, err := catalogue.GivenPorts(m, settings[m.Module])
|
|
if err != nil {
|
|
// A given port its definition no longer publishes: the module is left out of the
|
|
// declaration, by name, when it is composed (novox/hq ADR 0163, rule 6) — never the
|
|
// machine refused here for it.
|
|
continue
|
|
}
|
|
if g != nil {
|
|
given[m.Module] = g
|
|
}
|
|
}
|
|
// And one holder per machine port across the node's modules: settings written before this was
|
|
// refused where they are set are refused here rather than composed into two containers on
|
|
// one port.
|
|
holders := map[int]string{}
|
|
for _, m := range plan.Modules {
|
|
for _, at := range given[m.Module] {
|
|
if other, twice := holders[at]; twice && other != m.Module {
|
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+
|
|
"both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module,
|
|
at, node)
|
|
}
|
|
holders[at] = m.Module
|
|
}
|
|
}
|
|
|
|
ports := map[string]map[int]int{}
|
|
for _, m := range plan.Modules {
|
|
for _, l := range m.Listens {
|
|
if at, isGiven := given[m.Module][l.Port]; isGiven {
|
|
if ports[m.Module] == nil {
|
|
ports[m.Module] = map[int]int{}
|
|
}
|
|
ports[m.Module][l.Port] = at
|
|
continue
|
|
}
|
|
// **Only a port the module actually publishes is the mesh's to move.** A container's
|
|
// mapping is the thing that translates; without one the software binds what it binds,
|
|
// and an assignment would not move the service — it would open the wrong number in the
|
|
// rule set and leave the real one shut. Recorded either way, because this map means
|
|
// *where this module's port is on this machine* and every reader of it needs that
|
|
// answer whether or not the mesh was the one who chose it.
|
|
where, mayAssign := m.MachineSide(l.Port)
|
|
switch {
|
|
case mayAssign && choosing == Allocating:
|
|
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
|
"%s needs %d reachable on %s and it could not be assigned: %w",
|
|
m.Module, l.Port, node, err)
|
|
}
|
|
where = at.Machine
|
|
case mayAssign:
|
|
// Whatever was chosen last time, and nothing if there was no last time.
|
|
if at, known := already[m.Module][l.Port]; known {
|
|
where = at
|
|
}
|
|
}
|
|
if ports[m.Module] == nil {
|
|
ports[m.Module] = map[int]int{}
|
|
}
|
|
ports[m.Module][l.Port] = where
|
|
}
|
|
}
|
|
|
|
// And each module's own secrets — a superuser password, an administrator, an account. Made
|
|
// per node, so a module running on three machines has three.
|
|
needed := map[string]map[string]string{}
|
|
for _, m := range plan.Modules {
|
|
for name := range m.OwnSecrets {
|
|
// Minted on the send path and only read on every other. Making one is an insert, and
|
|
// a question that writes is a question that can block against the machine it is about.
|
|
var sealed string
|
|
var err error
|
|
if choosing == Allocating {
|
|
// **The broker credential is never invented here** (novox/hq issue 203). Every other
|
|
// own secret is the mesh's to make — a password nobody else knows — but this one
|
|
// is an account on the bus, minted by `module issue` and sealed by it; a push that
|
|
// made a random one would deliver a file the process cannot read and report the
|
|
// machine applied. Refused by name, with the verb.
|
|
if name == "broker" {
|
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
|
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
} else if !minted {
|
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
|
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
|
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
|
"`module issue %s --node %s`, then push again",
|
|
m.Module, node, user, m.Module, node)
|
|
}
|
|
}
|
|
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
|
} else {
|
|
var held bool
|
|
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
|
|
if err == nil && !held {
|
|
// Never issued, so this machine cannot be running it. Left out rather than
|
|
// invented: an empty string here would compose a declaration that differs
|
|
// from what would be sent, and the comparison this feeds would then be
|
|
// answering about a declaration nothing will ever push.
|
|
continue
|
|
}
|
|
}
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
if needed[m.Module] == nil {
|
|
needed[m.Module] = map[string]string{}
|
|
}
|
|
needed[m.Module][name] = sealed
|
|
}
|
|
}
|
|
// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
|
|
// rather than stored: the node's key does not change, so signing again produces an equally
|
|
// valid certificate and there is nothing to keep in step.
|
|
var certificate, authority string
|
|
for _, m := range plan.Modules {
|
|
if m.Certificate == nil {
|
|
continue
|
|
}
|
|
issued, meshCA, err := certificateFor(ctx, open, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
certificate, authority = issued, meshCA
|
|
break
|
|
}
|
|
|
|
// And who else is on the private network, which is what a rule saying "from the mesh"
|
|
// resolves to. Every node's address, including this one's: a machine reaching itself by its
|
|
// own overlay address rather than by loopback is ordinary, and leaving it out would filter
|
|
// the node's own traffic to itself with no rule naming why.
|
|
// One reading of the catalogue for the three questions below that resolve the whole mesh.
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
private, err := onThePrivateNetwork(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
// The private network's range, offered to a module as ${machine:mesh-range} — a module that must
|
|
// name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here
|
|
// rather than hardcoding a value it cannot know.
|
|
meshRange, err := overlayRange(ctx, inv)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
|
|
// The artifact store as this node reaches it now — the address every image and archive the
|
|
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
|
// built, so a reference recorded with an address before that is re-routed too.
|
|
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
held, err := inv.Held(ctx)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
built := make(map[string]bool, len(held))
|
|
for repository := range held {
|
|
built[repository] = true
|
|
}
|
|
|
|
// And every machine's name, so a container can reach one. The same set that writes the
|
|
// machine's own hosts file — one reading, so a container and its machine cannot disagree
|
|
// about where another machine is.
|
|
names, err := namesInTheMesh(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
|
|
// Each machine's operator account, so an ssh Host block can name the login for every node
|
|
// (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to.
|
|
allNodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
accounts := map[string]string{}
|
|
for _, n := range allNodes {
|
|
if n.Account != "" {
|
|
accounts[n.Name] = n.Account
|
|
}
|
|
}
|
|
|
|
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
|
|
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
|
|
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
|
|
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
|
|
// answer for any of them. The roster once carried every routed name, public ones included, and a
|
|
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
|
|
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
|
|
machines := make(map[string]string, len(names))
|
|
for name, at := range names {
|
|
machines[name] = at
|
|
}
|
|
|
|
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
|
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
|
// before it had an address on the private network. A machine joins through the tunnel now, and
|
|
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
|
|
// Nothing the mesh itself needs is opened beyond what a module declares.
|
|
var foundation []int
|
|
|
|
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
|
|
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
|
|
// export changes whenever any secret in the mesh is made or rotated, so the vault's declaration
|
|
// changes with it and the vault node is sent again: that is what keeps its copy current, and
|
|
// the cost is one two-table read per composition of the vault's node, in every mode.
|
|
var kept *catalogue.KeptExport
|
|
for _, m := range plan.Modules {
|
|
if m.Keeps == "" {
|
|
continue
|
|
}
|
|
if kept, err = inv.OperatorExport(ctx); err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
break
|
|
}
|
|
|
|
// Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and
|
|
// the declaration carries openings and the mesh's guard in place of a filter.
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
// And, on an adopted node, which modules were taken there: the guard is derived from those
|
|
// only (novox/hq ADR 0103).
|
|
var taken map[string]bool
|
|
if record.Adopted {
|
|
list, err := inv.Taken(ctx, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
taken = map[string]bool{}
|
|
for _, m := range list {
|
|
taken[m] = true
|
|
}
|
|
}
|
|
// Where this node put the foundation's servers, for the control plane's own connections
|
|
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
|
|
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
|
|
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
// The bus's user list, for the machine that runs the bus. Composed per push rather than kept,
|
|
// because it is a function of the mesh's records and a kept copy could disagree with them.
|
|
busUsers, err := composeBusUsers(ctx, inv, plan.Modules)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
memberships, err := inv.BusMemberships(ctx)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
// Which of this machine's links face outside, which is what the derived filter is written
|
|
// around (novox/hq ADR 0140). Reported by the machine, never set.
|
|
outwardLinks, err := inv.OutwardLinksOf(ctx, node)
|
|
if err != nil {
|
|
return catalogue.Rendering{}, inventory.Node{}, err
|
|
}
|
|
return catalogue.Rendering{
|
|
BusMembership: memberships[node],
|
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
|
Machines: machines,
|
|
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
|
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
|
BusUsers: busUsers,
|
|
}, record, nil
|
|
}
|
|
|
|
// certificateFor is what the mesh certifies about one machine's internal name.
|
|
//
|
|
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
|
|
// the machine and whether it is on the private network, `identity` holds the authority and the
|
|
// key that machine reported. The process holding both grants asks each for its part
|
|
// (novox/hq ADR 0008).
|
|
func certificateFor(ctx context.Context, open *stores, node string) (string, string, error) {
|
|
inv := open.inventory
|
|
ident, err := open.Identity(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
serving, err := ident.ServingKeyOf(ctx, record.ID)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if serving == "" {
|
|
// The machine joined before it had one, or never reported it. Said plainly, because the
|
|
// remedy is on the machine and no amount of pushing from here will produce one.
|
|
return "", "", fmt.Errorf(
|
|
"%s wants a certificate and has never told the mesh what key it serves with; it "+
|
|
"joins again to report one", node)
|
|
}
|
|
|
|
// The name it is certified for. Only a machine on the private network has one — a certificate
|
|
// for a name nothing resolves is a certificate nothing can check.
|
|
//
|
|
// With the catalogue, not without it. Being on the private network is a conclusion about what
|
|
// a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no
|
|
// network — which refused every certificate the mesh was asked for, and said the machine was
|
|
// not on a network it plainly was.
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
where, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
name := where[node]
|
|
if name == "" {
|
|
return "", "", fmt.Errorf(
|
|
"%s wants a certificate and is not on the private network, so it has no name inside "+
|
|
"the mesh to be certified for", node)
|
|
}
|
|
|
|
issued, err := ident.Certify(ctx, node, name, serving)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
authority, err := ident.EstablishAuthority(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
return issued, authority.Certificate, nil
|
|
}
|
|
|
|
// grantsFor is every credential this node must create, because something elsewhere uses it.
|
|
//
|
|
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
|
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
|
// the mesh hands over something it cannot itself use.
|
|
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
|
inv := open.inventory
|
|
issued, err := inv.SecretsFrom(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
|
// the mesh names machines.
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
|
// from a record beside it. A provider told to create a password and not what to create it for
|
|
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
|
out := make([]catalogue.Grant, 0, len(issued))
|
|
for _, s := range issued {
|
|
plan, settings, err := planFor(ctx, open, s.Consumer)
|
|
switch {
|
|
case unresolvable(err):
|
|
// Their set does not resolve. Skipped rather than fatal: this node is not the place
|
|
// to report another machine's problem, and a grant for something that is not going to
|
|
// run would have the provider create a user nothing uses.
|
|
continue
|
|
case err != nil:
|
|
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
|
// nothing — and withholding a grant on that reading takes a consumer's access away
|
|
// (novox/hq 04-ISSUES/152).
|
|
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
|
}
|
|
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// A port in there is the consumer's software port until this. The consumer is on another
|
|
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
|
// looked for in this one's, which is the whole reason the co-located fix could not reach
|
|
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
|
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
|
from := s.ConsumerModule
|
|
if !asks {
|
|
// That module no longer wants this. Left empty, which is what the declaration reads
|
|
// as "nobody asks for it any more" — and is how a login is withdrawn rather than kept
|
|
// working for ever after its consumer went away.
|
|
from = ""
|
|
}
|
|
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
|
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
|
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
|
// remedy a short slug rather than a login a provider silently shortened.
|
|
slug := ""
|
|
for _, mm := range plan.Modules {
|
|
if mm.Module == s.ConsumerModule {
|
|
slug = mm.Slug
|
|
break
|
|
}
|
|
}
|
|
if from != "" {
|
|
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
out = append(out, catalogue.Grant{
|
|
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
|
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// listensLines is what a person is told about what this module would open, and why — the same
|
|
// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so
|
|
// deciding whether to assign a module can see what it would open before it opens it, not only
|
|
// after. A module with nothing to listen on prints nothing extra, same as today.
|
|
func listensLines(m catalogue.Manifest) []string {
|
|
var out []string
|
|
for _, l := range m.Listens {
|
|
if l.Why == "" {
|
|
out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From))
|
|
continue
|
|
}
|
|
out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func planCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("plan", flag.ContinueOnError)
|
|
// Because "one resource" does not tell you whether the settings landed. Being able to read
|
|
// the file before it is sent is the difference between believing a merge worked and knowing.
|
|
show := set.Bool("files", false, "print the files this node would be given")
|
|
// The declaration exactly as the node would receive it. For handing to something else --
|
|
// checking it against the host's own parser, most usefully, which is the only way to know
|
|
// that what the control plane emits is what the host accepts.
|
|
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("plan <node> [--files] [--json]")
|
|
}
|
|
args = positionals
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
|
|
plan, settings, err := planFor(ctx, open, args[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(plan.Modules) == 0 {
|
|
fmt.Printf("%s is assigned nothing\n", args[0])
|
|
return nil
|
|
}
|
|
if *asJSON {
|
|
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an
|
|
// envelope other than the one sent.
|
|
body, err := declared.Body()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var indented bytes.Buffer
|
|
if err := json.Indent(&indented, body, "", " "); err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(indented.String())
|
|
return nil
|
|
}
|
|
|
|
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
|
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
|
// without --json allocates nothing.
|
|
if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil {
|
|
left := plan.LeftOut(settings, record.Adopted)
|
|
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
|
}
|
|
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
|
// stored before its definition moved from under it.
|
|
for _, m := range plan.Modules {
|
|
for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) {
|
|
fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray)
|
|
}
|
|
}
|
|
fmt.Printf("%s would run:\n", args[0])
|
|
for _, m := range plan.Modules {
|
|
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
|
for _, line := range listensLines(m) {
|
|
fmt.Println(line)
|
|
}
|
|
}
|
|
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
|
|
// one module on the wrong machine, the others still run, and the remedy is to move this one.
|
|
for _, u := range plan.Unhostable {
|
|
for _, c := range u.Missing {
|
|
fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0]))
|
|
}
|
|
}
|
|
for _, c := range plan.Claims {
|
|
fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope)
|
|
}
|
|
// What this machine depends on that is not on it. Worth saying out loud: it is the only part
|
|
// of a node's set that stops working when a *different* machine goes away, and nothing else
|
|
// in this output would have told anybody that.
|
|
for _, n := range plan.Needs {
|
|
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
|
}
|
|
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resources := declared.Resources
|
|
for module, layers := range settings {
|
|
for _, layer := range layers {
|
|
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
|
}
|
|
}
|
|
fmt.Printf("\n%d resource(s)\n", len(resources))
|
|
|
|
if *show {
|
|
for _, r := range resources {
|
|
content, ok := r["content"].(string)
|
|
if !ok {
|
|
continue
|
|
}
|
|
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// shownAs is the heading `plan --show` puts over a resource's content.
|
|
//
|
|
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
|
|
// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue
|
|
// 128). Shown under a bare path it reads as the whole file, and a person checking what a take
|
|
// replaces would see a hosts file of a dozen lines where the machine keeps thirty.
|
|
func shownAs(r map[string]any) string {
|
|
heading := fmt.Sprintf("%v %v", r["id"], r["path"])
|
|
if into, ok := r["into"].(string); ok && into != "" {
|
|
heading += fmt.Sprintf(" (written into, %s)", into)
|
|
}
|
|
return heading
|
|
}
|
|
|
|
// licencesFor is what this node can be answered with by record, and what it was put on.
|
|
//
|
|
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
|
|
// mesh is one, and a control plane that refused to plan because nobody had bought an API key
|
|
// would be unusable for the thing it already does.
|
|
func licencesFor(ctx context.Context, open *stores, node string) (
|
|
map[string][]catalogue.Record, map[string]map[string]catalogue.Record, error) {
|
|
|
|
held, err := open.Licences(ctx)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
all, err := held.All(ctx)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(all) == 0 {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
offered := map[string][]catalogue.Record{}
|
|
byName := map[string]catalogue.Record{}
|
|
for _, one := range all {
|
|
record := catalogue.Record{Name: one.Name, Serves: one.Serves}
|
|
offered[licences.Provision] = append(offered[licences.Provision], record)
|
|
byName[one.Name] = record
|
|
}
|
|
|
|
using := map[string]map[string]catalogue.Record{}
|
|
for _, one := range all {
|
|
holders, err := held.HoldersOf(ctx, one.Name)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for _, h := range holders {
|
|
if h.Node != node {
|
|
continue
|
|
}
|
|
if using[h.Module] == nil {
|
|
using[h.Module] = map[string]catalogue.Record{}
|
|
}
|
|
using[h.Module][licences.Provision] = byName[one.Name]
|
|
}
|
|
}
|
|
return offered, using, nil
|
|
}
|
|
|
|
// keyFor is the licence key sealed to one machine, for one module.
|
|
//
|
|
// **Empty is not an error here.** The mesh discarded the plaintext when it was supplied, so a
|
|
// holder recorded afterwards genuinely has no key — and the declaration refuses that by name,
|
|
// where the module and the path are both in view, rather than here.
|
|
func keyFor(ctx context.Context, open *stores, licence, node, module string) (string, error) {
|
|
held, err := open.Licences(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return held.KeyFor(ctx, licence, node, module)
|
|
}
|
|
|
|
// managerPublicKeyFor is the manager node's public sealing key, but only when (node, module) is the
|
|
// licence's manager holder — empty otherwise.
|
|
//
|
|
// It is delivered to the manager module in its bound facts so it can re-seal a rotated refresh token
|
|
// to this node (novox/hq ADR 0050). Public, so it travels in the clear like any other bound fact; and
|
|
// scoped to the manager holder alone, so a consumer never receives it and nothing invites a consumer
|
|
// to seal anything.
|
|
func managerPublicKeyFor(
|
|
ctx context.Context, open *stores, inv *inventory.Inventory, licence, node, module string,
|
|
) (string, error) {
|
|
held, err := open.Licences(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
managerNode, managerModule, err := held.ManagerOf(ctx, licence)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if managerNode == "" || node != managerNode || module != managerModule {
|
|
return "", nil
|
|
}
|
|
return inv.SealingKeyOf(ctx, node)
|
|
}
|
|
|
|
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
|
|
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
|
|
// assigned over the manifest's own, settled with the node's settings layers.
|
|
//
|
|
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
|
|
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
|
// given ports are that node's refusal to report, not this reader's.
|
|
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
|
m catalogue.Manifest, provision string) (map[string]any, error) {
|
|
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
serves := catalogue.ServedOn(m, provision, ports)
|
|
if len(serves) > 0 {
|
|
serves, err = catalogue.Settle(serves, layers)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return serves, nil
|
|
}
|
|
|
|
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
|
// node's settings layers for the module, read once for both.
|
|
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
|
|
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
|
|
ports, err := portsOn(ctx, inv, node, m.Module)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
|
for wanted, at := range given {
|
|
ports[wanted] = at
|
|
}
|
|
}
|
|
return ports, layers, nil
|
|
}
|
|
|
|
// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the
|
|
// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102).
|
|
//
|
|
// Read for every module in the catalogue that claims a seat, in this node's set or not: the store
|
|
// and the broker are given their ports at genesis, as settings on a module that may be registered
|
|
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
|
|
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
|
|
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
|
node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) {
|
|
assigned := map[string]bool{}
|
|
for _, m := range inSet {
|
|
assigned[m.Module] = true
|
|
}
|
|
names := make([]string, 0, len(shelf))
|
|
for name := range shelf {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
seats := map[string]map[int]int{}
|
|
for _, name := range names {
|
|
m := shelf[name]
|
|
var claims []string
|
|
for _, c := range m.Claims {
|
|
if c.At() == catalogue.ScopeMesh {
|
|
claims = append(claims, c.Name)
|
|
}
|
|
}
|
|
if len(claims) == 0 {
|
|
continue
|
|
}
|
|
// **Only a port the node was given or the mesh assigned — never the manifest's own
|
|
// number.** The sealed value the answer sits beside carries the port genesis wrote, which
|
|
// on a given-port node is the predecessor's; a manifest's long-form mapping is the
|
|
// catalogue's default, and answering with it would override the right number with one
|
|
// the mesh never checked (the contract in seat_into.go). And on an adopted node a holder
|
|
// assigned but not yet taken is the found container, on the ports it was found with, not
|
|
// the declaration's — so its mesh-assigned ports do not count there either; a given port
|
|
// does, because a given port is the found one by construction (ADR 0100).
|
|
ports, _, err := portsGivenOn(ctx, inv, node, m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if adopted && !taken[name] {
|
|
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ports = map[int]int{}
|
|
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
|
ports = given
|
|
}
|
|
}
|
|
if len(ports) == 0 {
|
|
continue
|
|
}
|
|
for _, seat := range claims {
|
|
if seats[seat] == nil {
|
|
seats[seat] = map[int]int{}
|
|
}
|
|
for wanted, at := range ports {
|
|
if _, said := seats[seat][wanted]; said && !assigned[name] {
|
|
continue
|
|
}
|
|
seats[seat][wanted] = at
|
|
}
|
|
}
|
|
}
|
|
return seats, nil
|
|
}
|
|
|
|
// portsOn is one module's assignments on one machine, by the port the software uses.
|
|
func portsOn(
|
|
ctx context.Context, inv *inventory.Inventory, node, module string,
|
|
) (map[int]int, error) {
|
|
all, err := inv.PortsFor(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[int]int{}
|
|
for _, a := range all {
|
|
if a.Module == module {
|
|
out[a.Wanted] = a.Machine
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// composeBusUsers is the bus's user list, for a push to the machine that runs the bus.
|
|
//
|
|
// Empty for every other machine, and for every machine while the mesh is on the bus it runs on
|
|
// today — where accounts are a management call and there is no file to write.
|
|
//
|
|
// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists,
|
|
// what runs where, what each declares), and a stored copy would be a second account of who may reach
|
|
// the bus, able to disagree with the records while both looked internally consistent (ADR 0043).
|
|
//
|
|
// A user the mesh has never minted a password for is **left out and said**, not written as a user
|
|
// without one — the composer refuses that, because a user with no password is a user anybody is. That
|
|
// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries
|
|
// the rest rather than failing: a bus that is missing one module's user is a mesh where that module
|
|
// cannot connect, and a bus with no file at all is a mesh where nothing can.
|
|
func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
|
onThisNode []catalogue.Manifest) (string, error) {
|
|
|
|
// **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only
|
|
// once the mesh is on the new bus" — which cannot work, because the server needs its user list
|
|
// *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the
|
|
// mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq
|
|
// ADR 0116). Under the old gating that step could not happen: the module would come up, find no
|
|
// accounts file, and its entrypoint would wait for one the controller had decided not to write.
|
|
//
|
|
// So the question is only whether this machine runs the module that asked for the file. A mesh
|
|
// that never moves has written a user list nothing reads, which costs a few hundred bytes on one
|
|
// node; the reverse cost a step that cannot be taken.
|
|
//
|
|
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
|
// resource of that module, so the question is whether it is here.
|
|
holdsTheBus := false
|
|
for _, m := range onThisNode {
|
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
|
holdsTheBus = true
|
|
}
|
|
}
|
|
if !holdsTheBus {
|
|
return "", nil
|
|
}
|
|
|
|
records, err := inv.BusRecords(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
users, err := broker.Users(records)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
kept, err := inv.BusUsers(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
hashes := make(map[string]string, len(kept))
|
|
for name, u := range kept {
|
|
hashes[name] = u.PasswordHash
|
|
}
|
|
filled, missing := broker.WithPasswords(users, hashes)
|
|
if len(missing) > 0 {
|
|
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
|
"for: %s. Each is a user that cannot connect until one is issued\n",
|
|
len(missing), strings.Join(missing, ", "))
|
|
}
|
|
if len(filled) == 0 {
|
|
return "", fmt.Errorf(
|
|
"this machine runs the bus and not one user has a credential, so the composed list " +
|
|
"would refuse every connection in the mesh")
|
|
}
|
|
return broker.ComposeAccounts(filled)
|
|
}
|
|
|
|
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
|
// own set when the provider is here, else the one the catalogue says offers it.
|
|
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
|
for _, m := range resolved.Modules {
|
|
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
|
return m.Module
|
|
}
|
|
}
|
|
shelf, err := open.inventory.Catalogue(ctx)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
for name, m := range shelf {
|
|
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
|
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
|
//
|
|
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
|
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
|
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
|
// burying the line about the node it acted on. A command says what concerns its own act instead
|
|
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
|
var (
|
|
unheldLogged = map[string]string{}
|
|
unheldLoggedMu sync.Mutex
|
|
logUnheldChanges bool
|
|
)
|
|
|
|
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
|
// it, including when they become none — in the serving controller, and nowhere else.
|
|
func logUnheld(node string, unheld []catalogue.Unheld) {
|
|
if !logUnheldChanges {
|
|
return
|
|
}
|
|
lines := make([]string, 0, len(unheld))
|
|
for _, u := range unheld {
|
|
lines = append(lines, u.String())
|
|
}
|
|
now := strings.Join(lines, "\n")
|
|
unheldLoggedMu.Lock()
|
|
before, seen := unheldLogged[node]
|
|
unheldLogged[node] = now
|
|
unheldLoggedMu.Unlock()
|
|
if (seen && before == now) || (!seen && now == "") {
|
|
return
|
|
}
|
|
if now == "" {
|
|
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
|
return
|
|
}
|
|
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
|
node, len(lines), strings.Join(lines, "\n "))
|
|
}
|
|
|
|
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
|
|
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
|
|
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
|
// any other node, and nothing that was already true before the act.
|
|
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
|
judge := func(names []string) map[string]catalogue.Unheld {
|
|
var set []catalogue.Manifest
|
|
for _, n := range names {
|
|
if m, known := shelf[n]; known {
|
|
set = append(set, m)
|
|
}
|
|
}
|
|
out := map[string]catalogue.Unheld{}
|
|
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
|
out[u.Module+" "+u.Seat] = u
|
|
}
|
|
return out
|
|
}
|
|
was, now := judge(before), judge(after)
|
|
var lines []string
|
|
for _, k := range sortedNames(now) {
|
|
if _, already := was[k]; !already {
|
|
lines = append(lines, "but "+now[k].String())
|
|
}
|
|
}
|
|
for _, k := range sortedNames(was) {
|
|
if _, still := now[k]; still {
|
|
continue
|
|
}
|
|
u := was[k]
|
|
if !slices.Contains(after, u.Module) {
|
|
continue // went with its module, which says nothing about the seat
|
|
}
|
|
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
|
}
|
|
return lines
|
|
}
|
|
|
|
func sortedNames[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|