diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 3813d03b..0f479621 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -153,499 +153,6 @@ func reportLeftOut(node string, declared sendable) { } } -// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued. -// -// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is -// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted -// by `module issue` and sealed by it; a push that made a random one would deliver a file the process -// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on -// a question asked ahead of it (Foreseeing), so that one is never told the push will make it. -func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error { - var user string - switch name { - case "broker": - user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() - case broker.ProvisionerSecret: - // And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11, - // design 57 P3): the same refusal, for the same reason. - user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username() - default: - return nil - } - if _, minted, err := inv.BusUserHash(ctx, user); err != nil { - return err - } else if !minted { - return fmt.Errorf( - "%s on %s has no bus credential: nothing was issued for %s, and a push "+ - "would seal a placeholder its process cannot read (novox/hq issue 203). "+ - "`module issue %s --node %s`, then push again", - module, node, user, module, node) - } - return nil -} - -// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside -// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again. -func givenUnderAnOldKey(module, node, name string) error { - return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ - "since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node) -} - -// renderingFor is everything a node's declaration is composed with, and the node's record. -func renderingFor(ctx context.Context, open *stores, node string, - plan catalogue.Resolution, settings catalogue.SettingsBy, - gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) { - inv := open.inventory - grants, withheld, unbound, err := grantsFor(ctx, open, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // Where this machine puts what each module needs reachable (novox/hq ADR 0038). - // - // **Assigned here rather than written by a module**, because a module is written once and - // assigned anywhere: any number it picks is a guess about a machine it has never seen. Made - // before the declaration is composed, because the container's mapping, the rule set and what a - // consumer is told are all derived from it. - // What this machine was already given, for a composition that may not allocate. - already := map[string]map[int]int{} - if choosing != Allocating { - held, err := inv.PortsFor(ctx, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - for _, a := range held { - if already[a.Module] == nil { - already[a.Module] = map[int]int{} - } - already[a.Module][a.Wanted] = a.Machine - } - } - - // The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's - // ports, as genesis chose them. A given port wins over anything assigned and over a manifest's - // own long-form mapping. - given := map[string]map[int]int{} - for _, m := range plan.Modules { - g, err := catalogue.GivenPorts(m, settings[m.Module]) - if err != nil { - // A given port its definition no longer publishes: the module is left out of the - // declaration, by name, when it is composed (novox/hq ADR 0163, rule 6) — never the - // machine refused here for it. - continue - } - if g != nil { - given[m.Module] = g - } - } - // And one holder per machine port across the node's modules: settings written before this was - // refused where they are set are refused here rather than composed into two containers on - // one port. - holders := map[int]string{} - for _, m := range plan.Modules { - for _, at := range given[m.Module] { - if other, twice := holders[at]; twice && other != m.Module { - return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+ - "both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module, - at, node) - } - holders[at] = m.Module - } - } - - ports := map[string]map[int]int{} - for _, m := range plan.Modules { - for _, l := range m.Listens { - if at, isGiven := given[m.Module][l.Port]; isGiven { - if ports[m.Module] == nil { - ports[m.Module] = map[int]int{} - } - ports[m.Module][l.Port] = at - continue - } - // **Only a port the module actually publishes is the mesh's to move.** A container's - // mapping is the thing that translates; without one the software binds what it binds, - // and an assignment would not move the service — it would open the wrong number in the - // rule set and leave the real one shut. Recorded either way, because this map means - // *where this module's port is on this machine* and every reader of it needs that - // answer whether or not the mesh was the one who chose it. - where, mayAssign := m.MachineSide(l.Port) - switch { - case mayAssign && choosing == Allocating: - at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf( - "%s needs %d reachable on %s and it could not be assigned: %w", - m.Module, l.Port, node, err) - } - where = at.Machine - case mayAssign: - // Whatever was chosen last time, and nothing if there was no last time. - if at, known := already[m.Module][l.Port]; known { - where = at - } - } - if ports[m.Module] == nil { - ports[m.Module] = map[int]int{} - } - ports[m.Module][l.Port] = where - } - } - - // And each module's own secrets — a superuser password, an administrator, an account. Made - // per node, so a module running on three machines has three. - needed := map[string]map[string]string{} - foreseen := map[string]map[string]bool{} - // **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11, - // design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for - // this machine. Every other module's are made and sealed as before. - hands, err := engineTakesHanded(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)} - for _, m := range plan.Modules { - if handing.Handing(m) { - continue - } - // **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is - // placed, and one not given is nothing — the module says it waits for it. - for _, family := range m.OwnSecrets.Families() { - members, err := inv.GivenMembers(ctx, node, m.Module, family) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - for _, g := range members { - if _, fam, ok := m.OwnSecrets.Lookup(g.Name); !ok || fam != family { - continue // a longer family's member, or a name no longer of this family - } - if !g.Current { - if choosing == Allocating { - return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name) - } - continue - } - if needed[m.Module] == nil { - needed[m.Module] = map[string]string{} - } - needed[m.Module][g.Name] = g.Sealed - } - } - for name, own := range m.OwnSecrets.Plain() { - // **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for - // a family): only a value a person gave is placed. One not given is nothing — no file, and an empty - // value where a file of the module names it — so the module can tell it was not given, where a value - // the mesh made looked given and wrong. A value the mesh made for it before is not one either. - if own.IssuedBy == catalogue.IssuedOutside { - g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - if !given { - continue - } - if !g.Current { - if choosing == Allocating { - return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name) - } - continue - } - if needed[m.Module] == nil { - needed[m.Module] = map[string]string{} - } - needed[m.Module][name] = g.Sealed - continue - } - // Minted on the send path and only read on every other. Making one is an insert, and - // a question that writes is a question that can block against the machine it is about. - var sealed string - var err error - if choosing == Allocating { - if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - sealed, err = inv.SecretForModule(ctx, node, m.Module, name) - } else { - var held bool - sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name) - if err == nil && !held && choosing == Foreseeing { - // Asked ahead of the send: what the send would do about it, by the send's own - // rules. Made by it — a stand-in, named; refused by it — refused here, the same - // words (novox/hq issue 275). - if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - if foreseen[m.Module] == nil { - foreseen[m.Module] = map[string]bool{} - } - foreseen[m.Module][name] = true - continue - } - if err == nil && !held { - // Never issued, so this machine cannot be running it. Left out rather than - // invented: an empty string here would compose a declaration that differs - // from what would be sent, and the comparison this feeds would then be - // answering about a declaration nothing will ever push. - continue - } - } - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - if needed[m.Module] == nil { - needed[m.Module] = map[string]string{} - } - needed[m.Module][name] = sealed - } - } - // **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that - // opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that - // contract nothing here is asked and the composition is as it was. - var core coreBoxes - opensBox, err := engineOpensVaultBox(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - if opensBox { - core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool { - return handing.Handing(m) - }, choosing) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - } - // And a certificate for this machine's name inside the mesh, when anything on it asks. Issued - // rather than stored: the node's key does not change, so signing again produces an equally - // valid certificate and there is nothing to keep in step. - var certificate, authority string - for _, m := range plan.Modules { - if m.Certificate == nil { - continue - } - issued, meshCA, err := certificateFor(ctx, open, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - certificate, authority = issued, meshCA - break - } - - // And who else is on the private network, which is what a rule saying "from the mesh" - // resolves to. Every node's address, including this one's: a machine reaching itself by its - // own overlay address rather than by loopback is ordinary, and leaving it out would filter - // the node's own traffic to itself with no rule naming why. - // One reading of the catalogue for the three questions below that resolve the whole mesh. - shelf, err := inv.Catalogue(ctx) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - private, err := onThePrivateNetwork(ctx, inv, shelf) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // The private network's range, offered to a module as ${machine:mesh-range} — a module that must - // name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here - // rather than hardcoding a value it cannot know. - meshRange, err := overlayRange(ctx, inv) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - - // The artifact store as this node reaches it now — the address every image and archive the - // mesh built is fetched through, composed here and recorded nowhere — with what the mesh has - // built, so a reference recorded with an address before that is re-routed too. - artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - held, err := inv.Held(ctx) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - built := make(map[string]bool, len(held)) - for repository := range held { - built[repository] = true - } - - // And every machine's name, so a container can reach one. The same set that writes the - // machine's own hosts file — one reading, so a container and its machine cannot disagree - // about where another machine is. - names, err := namesInTheMesh(ctx, inv, shelf) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - - // Each machine's operator account, so an ssh Host block can name the login for every node - // (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to. - allNodes, err := inv.Nodes(ctx) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - accounts := map[string]string{} - for _, n := range allNodes { - if n.Account != "" { - accounts[n.Name] = n.Account - } - } - - // **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal - // domain, `.internal`, and every route on it is a name under that domain (ADR 0151), which - // the resolver answers with one wildcard per machine — so no route needs a line of its own. A - // node's public domains are the operator's and public DNS answers them; the mesh gives no private - // answer for any of them. The roster once carried every routed name, public ones included, and a - // resolver that also serves a LAN handed a phone a tunnel address for the mail server. - // `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111). - machines := make(map[string]string, len(names)) - for name, at := range names { - machines[name] = at - } - - // And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver - // to forward. - zones, err := zonesInTheMesh(ctx, open) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - - // And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file - // lists every holder of the mesh's resolver. - replicas, err := replicatedHolders(ctx, inv, shelf) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - - // **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the - // broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol - // before it had an address on the private network. A machine joins through the tunnel now, and - // every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh. - // Nothing the mesh itself needs is opened beyond what a module declares. - var foundation []int - - // And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's - // copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The - // export changes whenever any secret in the mesh is made or rotated, so the vault's declaration - // changes with it and the vault node is sent again: that is what keeps its copy current, and - // the cost is one two-table read per composition of the vault's node, in every mode. - var kept *catalogue.KeptExport - for _, m := range plan.Modules { - if m.Keeps == "" { - continue - } - if kept, err = inv.OperatorExport(ctx); err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - break - } - - // Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and - // the declaration carries openings and the mesh's guard in place of a filter. - record, err := inv.NodeByName(ctx, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // And, on an adopted node, which modules were taken there: the guard is derived from those - // only (novox/hq ADR 0103). - var taken map[string]bool - if record.Adopted { - list, err := inv.Taken(ctx, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - taken = map[string]bool{} - for _, m := range list { - taken[m] = true - } - } - // Where this node put the foundation's servers, for the control plane's own connections - // (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat, - // exactly as a consumer's binding is, never from what genesis wrote into a secret. - seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // The bus's user list, for the machine that runs the bus. Composed per push rather than kept, - // because it is a function of the mesh's records and a kept copy could disagree with them. - busUsers, err := composeBusUsers(ctx, inv, plan.Modules) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - memberships, err := inv.BusMemberships(ctx) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // Which of this machine's links face outside, which is what the derived filter is written - // around (novox/hq ADR 0140). Reported by the machine, never set. - outwardLinks, err := inv.OutwardLinksOf(ctx, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // And which of the networks it is attached to count as its local network (novox/hq ADR 0340): - // private ranges only, on a link facing outside it. Reported by the machine, never set. - local, err := localNetworksOf(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - // Where this machine reaches each mesh seat's holder, for ${seat::reach} (novox/hq ADR - // 0222): the artifact store as this network reaches it, which the container runtime is told to - // trust (ADR 0082). The same address composed into every reference the mesh built. - var reach map[string]string - if artifactStore != "" { - reach = map[string]string{"mesh-artifact-store": artifactStore} - } - readsHealth, err := engineReadsHealth(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - judgesRoot, err := engineJudgesRoot(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - readsBusy, err := engineReadsBusy(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } - return catalogue.Rendering{ - ReadsHealth: readsHealth, - JudgesRoot: judgesRoot, - ReadsBusy: readsBusy, - Hands: hands, - Provisioner: provisionerAccount(plan), - BusMembership: memberships[node], - Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, - VaultSealed: core.sealed, NotPlaced: core.notPlaced, - Certificate: certificate, Authority: authority, Mesh: private, Names: names, - Machines: machines, Zones: zones, Holders: replicas, - Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation, - Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks, - LocalNetworks: catalogue.CountedLocalNetworks(local), - Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built, - BusUsers: busUsers, Withheld: withheld, Unbound: unbound, - }, record, nil -} - -// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240 -// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the -// field, because it parses strictly and would refuse the whole declaration for it. -func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return had && stated.Contract >= link.ReadinessContract, nil -} - -// engineJudgesRoot says whether a machine's node-engine judges a user's declared `root` (novox/hq ADR 0266), -// by its own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. -func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return had && stated.Contract >= link.RootContract, nil -} - // engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its // own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { @@ -656,23 +163,6 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) return had && stated.Contract >= link.BusyContract, nil } -// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone -// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the -// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before. -func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return takesHanded(stated, had), nil -} - -// takesHanded is whether a machine's newest statement (had: it ever stated one) is of an engine that writes a -// filled file on the provisioner client's word alone. -func takesHanded(stated inventory.NodeHealth, had bool) bool { - return had && stated.Contract >= link.HandedContract -} - // provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user // the process of the module running it runs as, empty where no such module is on the machine. func provisionerAccount(plan catalogue.Resolution) string { @@ -710,71 +200,6 @@ func busyNotSent(modules []catalogue.Manifest, reads bool) []string { return out } -// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR -// 0199): the zone settled from that node's settings, the node's private address, the port the -// answering listen is published on there. -// -// Read across every machine's resolution, as the roster once read routed names: a node whose set does -// not compose declares nothing and is passed over, so one broken machine does not cost the rest their -// zones; a store that cannot be read is raised, naming the machine, because returning the zones -// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152). -// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's -// suffix or a node's public domain — is refused here, by name. -func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) { - inv := open.inventory - places, err := inv.Overlays(ctx) - if err != nil { - return nil, fmt.Errorf("where the machines are cannot be read: %w", err) - } - address := map[string]string{} - for _, p := range places { - if strings.TrimSpace(p.Address) != "" { - address[p.Name] = p.Address - } - } - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err) - } - var zones []catalogue.ZoneAt - var public []string - for _, n := range nodes { - plan, _, err := resolveNode(ctx, open, n.Name) - switch { - case unresolvable(err): - continue - case err != nil: - return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err) - } - if plan.PublicDomain != "" { - public = append(public, plan.PublicDomain) - } - for _, m := range plan.Modules { - if m.Zone == nil { - continue - } - at := address[n.Name] - if at == "" { - // Not on the private network yet: nothing could reach its answerer. - continue - } - published, layers, err := portsGivenOn(ctx, inv, n.Name, m) - if err != nil { - return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err) - } - z, err := catalogue.ZoneOn(m, layers, published, n.Name, at) - if err != nil { - return nil, err - } - zones = append(zones, *z) - } - } - if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 { - return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - ")) - } - return zones, nil -} - // certificateFor is what the mesh certifies about one machine's internal name. // // It reaches across two contexts and reads neither one's store from the other: `inventory` knows