package catalogue // Turning a resolution into the declaration a node is sent. // // Separate from resolving because they answer different questions. Resolving asks *what should // this machine run*; this asks *what does that look like as resources*, and the second is where // settings are applied, generators are called, contributions are collected and credentials are // placed. Both lived in one file until it was doing four jobs at once — which is the shape the // system this replaces failed in, one import at a time. import ( "encoding/json" "fmt" "sort" "strings" ) // SettingsBy is the layers that apply to each module, keyed by module name. type SettingsBy map[string][]Layer // Generator works out a module's resources for one node, where they cannot be written in advance. type Generator interface { // Resources for this node. Absent means the node is not part of whatever this generates, // which is an ordinary answer rather than a failure — a machine assigned the module before it // has an address on the network is in exactly that state. Resources(node string) ([]map[string]any, bool, error) } // Grant is one consumer's credential, on the machine that must create it. type Grant struct { // Provision is what was required. Provision string // Consumer is the node that will use it, which is also what names the file. Consumer string // From is the module on that machine which asked, so the provider can name what it creates // after the thing using it rather than after the machine. From string // Values are what that module contributed — the name it wants, and anything else the // provision's own vocabulary defines. Values map[string]any // Sealed is the credential, closed to the providing node. Sealed string } // Rendering is everything needed to turn a resolution into the declaration a node is sent. type Rendering struct { // Certificate is what the mesh issued for this machine's internal name, and the mesh's own // certificate. Both public — the key they belong to never left the machine. Certificate string Authority string // Needed is each module's own secrets, sealed to this node, keyed by module and then by the // name the module gave it. Needed map[string]map[string]string // Mesh is every node's address on the private network, which is what a rule saying "from the // mesh" resolves to. Passed in for the same reason grants are: who else is on the network is // a fact about the mesh, and resolution answers questions about one machine. Mesh []string Settings SettingsBy Generators map[string]Generator // Grants are the credentials this node must create, for the provisions it offers. Passed in // rather than resolved, because who consumes a node is a fact about the rest of the mesh and // resolution answers questions about one machine. Grants []Grant } // Declaration is everything the resolved modules put on the node, with settings applied. // // Resource identities are prefixed with the module they came from. Two modules may reasonably // both call something "config", and without this the second would silently replace the first — // the node applying one of them and reporting success. func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // Where each provision's credentials land, so a contribution can name the file rather than // carry a value the mesh does not have. directories := map[string]string{} for _, m := range r.Modules { for provision, where := range m.Grants { directories[provision] = where } } given, err := r.contributions(with.Settings, with.Grants, directories) if err != nil { return nil, err } // Once, from every module's listens -- not per module. A module receiving only its own ports // would write a rule set that closed every other module on the machine. filtering := AsNftables(r.Filtering(), with.Mesh) var out []map[string]any for _, m := range r.Modules { resources := m.Resources // What the mesh computes for this module goes FIRST, before the module's own resources. // // **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so // whatever the mesh writes down is the order a machine applies. A module's service or // container routinely depends on one of these files; nothing here ever depends on a // module's resources, because none of it is computed from them. // // Appended, this was wrong in a way that only showed on the first apply and then healed: // the service started before its certificate or its rule set existed, failed, and the next // reconcile fixed it. A fault that repairs itself on the second attempt is worse than one // that does not, because what gets remembered is that it works. var first []map[string]any if f := m.Filtering; f != nil { first = append(first, map[string]any{ "id": FilteringID(), "type": "file", "path": f.Into, "content": filtering, "mode": "0600", }) } if c := m.Certificate; c != nil { if with.Certificate == "" { // Asked for and not issued. Refused rather than skipped: a module that serves TLS // with no certificate does not start, and the reason is somewhere else entirely. return nil, fmt.Errorf( "%s wants a certificate for this machine and none was issued", m.Module) } first = append(first, map[string]any{ "id": CertificateID(), "type": "file", "path": c.Into, // Public. It travels in the open like any other file, because it is a statement // about a key rather than the key. "content": with.Certificate, "mode": "0644", }) if c.Authority != "" { first = append(first, map[string]any{ "id": AuthorityID(), "type": "file", "path": c.Authority, "content": with.Authority, "mode": "0644", }) } } for _, name := range sortedKeys(m.Needs) { sealed := with.Needed[m.Module][name] if sealed == "" { // Declared and not made. Refused rather than skipped: a module whose own // credential is silently absent starts, fails to authenticate, and the reason is // three layers away from the machine reporting it. return nil, fmt.Errorf( "%s needs a secret called %q and none was made for it", m.Module, name) } first = append(first, map[string]any{ "id": NeedID(name), "type": "file", "path": m.Needs[name], "sealed": sealed, }) } for _, to := range sortedKeys(m.Secrets) { var found *Needed for i, n := range r.Needs { if n.Name == to { found = &r.Needs[i] } } if found == nil || found.Sealed == "" { // Answered on this machine, or answered by a node the mesh could not seal to. // Nothing to write either way, and writing an empty credential file would be // worse than none: something would read it and fail authenticating. continue } first = append(first, map[string]any{ "id": SecretID(to), "type": "file", "path": m.Secrets[to], "sealed": found.Sealed, }) } for _, to := range sortedKeys(m.Grants) { for _, g := range with.Grants { if g.Provision != to { continue } if g.From == "" { // Nothing on that machine asks for this any more. Skipped here rather than // where grants are gathered, so the rule holds whoever gathers them. // // **This is how a credential is withdrawn.** The provisioner removes what // nobody asks for, and it can only do that if the mesh stops asking — a // consumer that was unassigned would otherwise keep a working login for ever, // and nothing would say so. continue } first = append(first, map[string]any{ "id": GrantID(to, g.Consumer), "type": "file", "path": grantPath(m.Grants[to], g.Consumer), "sealed": g.Sealed, }) } } for _, to := range sortedKeys(m.Binds) { var found *Needed for i, n := range r.Needs { if n.Name == to { found = &r.Needs[i] } } if found == nil { // Bound to something answered on this machine rather than from the mesh. Nothing // to write: the answer is here, and a file saying "it is on this node" would be // a fact nobody needs and one more thing to keep true. continue } file, err := boundFile(*found, m.Binds[to]) if err != nil { return nil, err } first = append(first, file) } for _, to := range sortedKeys(m.Receives) { file, err := receivedFile(to, m.Receives[to], given[to]) if err != nil { return nil, err } first = append(first, file) } if m.Computed != "" { generator, known := with.Generators[m.Computed] if !known { return nil, fmt.Errorf( "%s says its resources are computed by %q, and this control plane has no %q", m.Module, m.Computed, m.Computed) } generated, part, err := generator.Resources(r.Node) if err != nil { return nil, err } if !part { // Assigned, and not yet part of what this generates. Nothing to put on the // machine, which is different from an error: a node given the network module // before it has an address is in exactly that state, briefly. continue } resources = generated } // Now, and not before: a module whose resources are computed replaces them wholesale, and // merging earlier would throw away the files it still needs. resources = append(append([]map[string]any{}, first...), resources...) for _, unsettled := range resources { resource, err := ApplySettings(unsettled, with.Settings[m.Module]) if err != nil { return nil, err } copied := map[string]any{} for k, v := range resource { copied[k] = v } copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those // are prefixed too or they would point at nothing. if reflects, ok := resource["restart-on"].([]any); ok { var renamed []any for _, id := range reflects { renamed = append(renamed, m.Module+"."+fmt.Sprint(id)) } copied["restart-on"] = renamed } out = append(out, copied) } } return out, nil } // Contribution is one module telling the answer to a requirement what it needs from it. type Contribution struct { // From is the module that said it, so the provider and a person reading the file can tell // which route belongs to what. From string `json:"from"` // Node is the machine it said it from, empty when that is this one. // // A provision answered from anywhere in the mesh has consumers on other machines, and the // provider has to know who they are — a database told to create a password and not who for // cannot do anything with it. Contributions were node-local until this, which meant the one // case that most needed them was the one they did not reach. Node string `json:"node,omitempty"` // Secret is the file on this machine holding that consumer's credential, sealed to it. // // Named rather than carried, for the same reason the private network's key is: the mesh // discarded the value and could not put it here if it wanted to. What is here is where to // find it. Secret string `json:"secret,omitempty"` // Values are the module's own, with settings applied. What the keys mean is agreed by the // requirement's name — everything providing `reverse-proxy` understands the same shape, which // is what makes swapping one for another cost nothing. Values map[string]any `json:"values"` } // grantPath is where one consumer's sealed credential lands on the providing machine. // // Suffixed, so the directory can also hold whatever the module writing it keeps there and so a // node named like something else in that directory cannot collide with it. func grantPath(directory, consumer string) string { return strings.TrimRight(directory, "/") + "/" + consumer + ".secret" } // contributions collects what every module in this set contributes, by requirement. // // Ordered by contributing module, because the result becomes a file on a machine and a file whose // lines move about is a file that looks changed when nothing changed. func (r Resolution) contributions(settings SettingsBy, grants []Grant, directories map[string]string) (map[string][]Contribution, error) { out := map[string][]Contribution{} modules := append([]Manifest{}, r.Modules...) sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module }) // What consumers on other machines asked for. Merged in with this machine's own, because from // the provider's side they are the same thing — somebody wanting something — and a provider // that had to read two lists would be a provider that reads one of them. sorted := append([]Grant{}, grants...) sort.Slice(sorted, func(i, j int) bool { if sorted[i].Provision != sorted[j].Provision { return sorted[i].Provision < sorted[j].Provision } return sorted[i].Consumer < sorted[j].Consumer }) for _, g := range sorted { if g.From == "" { // As above: nothing on that machine asks for this any more, so the provider is not // told about it and withdraws the login on its next pass. continue } out[g.Provision] = append(out[g.Provision], Contribution{ From: g.From, Node: g.Consumer, Values: g.Values, Secret: grantPath(directories[g.Provision], g.Consumer), }) } for _, m := range modules { for _, to := range sortedKeys(m.Contributes) { // Settings reach a contribution the same way they reach a file. A route's hostname is // exactly the kind of thing that differs between one mesh and the next, and a module // that could not have it set would have to be edited to be reused. values, err := settle(m.Contributes[to], settings[m.Module], nil, m.Module+" contributing to "+to) if err != nil { return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err) } out[to] = append(out[to], Contribution{From: m.Module, Values: values}) } } return out, nil } // receivedFile is the file a provider is given its consumers' contributions in. func receivedFile(requirement, path string, given []Contribution) (map[string]any, error) { if given == nil { // Nobody contributed. The file is still written, empty, rather than left absent: a // provider that finds no file cannot tell "nothing asked for me" from "the mesh never // wrote it", and the two want completely different responses. given = []Contribution{} } // The note goes *inside* the document, not above it. The first version wrote a `//` header // and produced a file that says "do not edit" to a person and fails to parse for the program // meant to read it — which is the whole audience. body, err := json.MarshalIndent(map[string]any{ "contributions": 1, "requirement": requirement, "generated": "by the mesh — do not edit; replaced whenever a module contributing to " + requirement + " arrives or leaves", "given": given, }, "", " ") if err != nil { return nil, err } return map[string]any{ "id": ReceivedID(requirement), "type": "file", "path": path, "mode": "0644", "content": string(body) + "\n", }, nil } // sortedKeys is map iteration made repeatable, which everything written to a machine needs. func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // boundFile is what a module is told about something it requires from another machine. // // Where it is and what the providing module said about using it. **No credential**, and the file // says so rather than leaving a reader to wonder whether one was meant to be there — a missing // field looks like a bug, and a stated absence looks like a boundary. func boundFile(n Needed, path string) (map[string]any, error) { body, err := json.MarshalIndent(map[string]any{ "binding": 1, "provision": n.Name, "from": n.From, "at": n.At, "serves": n.Serves, "generated": "by the mesh — do not edit; replaced whenever this changes. " + "It carries no credential: the mesh has no way to issue one yet", }, "", " ") if err != nil { return nil, err } return map[string]any{ "id": BoundID(n.Name), "type": "file", "path": path, "mode": "0644", "content": string(body) + "\n", }, nil } // ContributionsTo is what this node's set asked of one requirement, settled. // // Exported because a provider's grants are assembled from its consumers' resolutions, one machine // at a time, and the alternative was for the control plane to reimplement settling. func (r Resolution) ContributionsTo(requirement string, settings SettingsBy) ( string, map[string]any, error) { all, err := r.contributions(settings, nil, nil) if err != nil { return "", nil, err } given := all[requirement] if len(given) == 0 { return "", nil, nil } if len(given) > 1 { // Two modules on one machine wanting the same provision would share one credential, and // the provider would be told to create one thing under two names. Refused rather than // resolved by picking, which is the rule everywhere else here. var who []string for _, g := range given { who = append(who, g.From) } sort.Strings(who) return "", nil, fmt.Errorf( "%s has %d modules asking for %q and they would share one credential: %s", r.Node, len(given), requirement, strings.Join(who, ", ")) } return given[0].From, given[0].Values, nil }