jschoubben
1eb1b69cae
The host does not sort — order is stated (novox/hq ADR 0005) — so the order the mesh writes down is the order a machine applies. Certificates, credentials, bound files and the rule set were appended after a module's own resources, so a service or container that depends on one was applied before it existed. It failed and the next reconcile fixed it, which is why nothing caught it. A fault that repairs itself on the second attempt is worse than one that does not: what gets remembered is that it works. Nothing the mesh computes depends on a module's resources, so putting all of it first is unconditionally right. Merged after the computed-resources branch, which replaces a module's resources wholesale and would otherwise discard them.
432 lines
17 KiB
Go
432 lines
17 KiB
Go
package catalogue
|
|
|
|
// Turning a resolution into the declaration a node is sent.
|
|
//
|
|
// Separate from resolving because they answer different questions. Resolving asks *what should
|
|
// this machine run*; this asks *what does that look like as resources*, and the second is where
|
|
// settings are applied, generators are called, contributions are collected and credentials are
|
|
// placed. Both lived in one file until it was doing four jobs at once — which is the shape the
|
|
// system this replaces failed in, one import at a time.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// SettingsBy is the layers that apply to each module, keyed by module name.
|
|
type SettingsBy map[string][]Layer
|
|
|
|
// Generator works out a module's resources for one node, where they cannot be written in advance.
|
|
type Generator interface {
|
|
// Resources for this node. Absent means the node is not part of whatever this generates,
|
|
// which is an ordinary answer rather than a failure — a machine assigned the module before it
|
|
// has an address on the network is in exactly that state.
|
|
Resources(node string) ([]map[string]any, bool, error)
|
|
}
|
|
|
|
// Grant is one consumer's credential, on the machine that must create it.
|
|
type Grant struct {
|
|
// Provision is what was required.
|
|
Provision string
|
|
// Consumer is the node that will use it, which is also what names the file.
|
|
Consumer string
|
|
// From is the module on that machine which asked, so the provider can name what it creates
|
|
// after the thing using it rather than after the machine.
|
|
From string
|
|
// Values are what that module contributed — the name it wants, and anything else the
|
|
// provision's own vocabulary defines.
|
|
Values map[string]any
|
|
// Sealed is the credential, closed to the providing node.
|
|
Sealed string
|
|
}
|
|
|
|
// Rendering is everything needed to turn a resolution into the declaration a node is sent.
|
|
type Rendering struct {
|
|
// Certificate is what the mesh issued for this machine's internal name, and the mesh's own
|
|
// certificate. Both public — the key they belong to never left the machine.
|
|
Certificate string
|
|
Authority string
|
|
|
|
// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
|
|
// name the module gave it.
|
|
Needed map[string]map[string]string
|
|
|
|
// Mesh is every node's address on the private network, which is what a rule saying "from the
|
|
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
|
|
// a fact about the mesh, and resolution answers questions about one machine.
|
|
Mesh []string
|
|
|
|
Settings SettingsBy
|
|
Generators map[string]Generator
|
|
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
|
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
|
// resolution answers questions about one machine.
|
|
Grants []Grant
|
|
}
|
|
|
|
// Declaration is everything the resolved modules put on the node, with settings applied.
|
|
//
|
|
// Resource identities are prefixed with the module they came from. Two modules may reasonably
|
|
// both call something "config", and without this the second would silently replace the first —
|
|
// the node applying one of them and reporting success.
|
|
func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
|
// Where each provision's credentials land, so a contribution can name the file rather than
|
|
// carry a value the mesh does not have.
|
|
directories := map[string]string{}
|
|
for _, m := range r.Modules {
|
|
for provision, where := range m.Grants {
|
|
directories[provision] = where
|
|
}
|
|
}
|
|
given, err := r.contributions(with.Settings, with.Grants, directories)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Once, from every module's listens -- not per module. A module receiving only its own ports
|
|
// would write a rule set that closed every other module on the machine.
|
|
filtering := AsNftables(r.Filtering(), with.Mesh)
|
|
|
|
var out []map[string]any
|
|
for _, m := range r.Modules {
|
|
resources := m.Resources
|
|
|
|
// What the mesh computes for this module goes FIRST, before the module's own resources.
|
|
//
|
|
// **Order is stated, not derived — the host does not sort** (novox/hq ADR 0005), so
|
|
// whatever the mesh writes down is the order a machine applies. A module's service or
|
|
// container routinely depends on one of these files; nothing here ever depends on a
|
|
// module's resources, because none of it is computed from them.
|
|
//
|
|
// Appended, this was wrong in a way that only showed on the first apply and then healed:
|
|
// the service started before its certificate or its rule set existed, failed, and the next
|
|
// reconcile fixed it. A fault that repairs itself on the second attempt is worse than one
|
|
// that does not, because what gets remembered is that it works.
|
|
var first []map[string]any
|
|
if f := m.Filtering; f != nil {
|
|
first = append(first, map[string]any{
|
|
"id": FilteringID(), "type": "file", "path": f.Into,
|
|
"content": filtering, "mode": "0600",
|
|
})
|
|
}
|
|
if c := m.Certificate; c != nil {
|
|
if with.Certificate == "" {
|
|
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
|
|
// with no certificate does not start, and the reason is somewhere else entirely.
|
|
return nil, fmt.Errorf(
|
|
"%s wants a certificate for this machine and none was issued", m.Module)
|
|
}
|
|
first = append(first, map[string]any{
|
|
"id": CertificateID(), "type": "file", "path": c.Into,
|
|
// Public. It travels in the open like any other file, because it is a statement
|
|
// about a key rather than the key.
|
|
"content": with.Certificate, "mode": "0644",
|
|
})
|
|
if c.Authority != "" {
|
|
first = append(first, map[string]any{
|
|
"id": AuthorityID(), "type": "file", "path": c.Authority,
|
|
"content": with.Authority, "mode": "0644",
|
|
})
|
|
}
|
|
}
|
|
for _, name := range sortedKeys(m.Needs) {
|
|
sealed := with.Needed[m.Module][name]
|
|
if sealed == "" {
|
|
// Declared and not made. Refused rather than skipped: a module whose own
|
|
// credential is silently absent starts, fails to authenticate, and the reason is
|
|
// three layers away from the machine reporting it.
|
|
return nil, fmt.Errorf(
|
|
"%s needs a secret called %q and none was made for it", m.Module, name)
|
|
}
|
|
first = append(first, map[string]any{
|
|
"id": NeedID(name), "type": "file", "path": m.Needs[name], "sealed": sealed,
|
|
})
|
|
}
|
|
for _, to := range sortedKeys(m.Secrets) {
|
|
var found *Needed
|
|
for i, n := range r.Needs {
|
|
if n.Name == to {
|
|
found = &r.Needs[i]
|
|
}
|
|
}
|
|
if found == nil || found.Sealed == "" {
|
|
// Answered on this machine, or answered by a node the mesh could not seal to.
|
|
// Nothing to write either way, and writing an empty credential file would be
|
|
// worse than none: something would read it and fail authenticating.
|
|
continue
|
|
}
|
|
first = append(first, map[string]any{
|
|
"id": SecretID(to), "type": "file", "path": m.Secrets[to],
|
|
"sealed": found.Sealed,
|
|
})
|
|
}
|
|
for _, to := range sortedKeys(m.Grants) {
|
|
for _, g := range with.Grants {
|
|
if g.Provision != to {
|
|
continue
|
|
}
|
|
if g.From == "" {
|
|
// Nothing on that machine asks for this any more. Skipped here rather than
|
|
// where grants are gathered, so the rule holds whoever gathers them.
|
|
//
|
|
// **This is how a credential is withdrawn.** The provisioner removes what
|
|
// nobody asks for, and it can only do that if the mesh stops asking — a
|
|
// consumer that was unassigned would otherwise keep a working login for ever,
|
|
// and nothing would say so.
|
|
continue
|
|
}
|
|
first = append(first, map[string]any{
|
|
"id": GrantID(to, g.Consumer),
|
|
"type": "file",
|
|
"path": grantPath(m.Grants[to], g.Consumer),
|
|
"sealed": g.Sealed,
|
|
})
|
|
}
|
|
}
|
|
for _, to := range sortedKeys(m.Binds) {
|
|
var found *Needed
|
|
for i, n := range r.Needs {
|
|
if n.Name == to {
|
|
found = &r.Needs[i]
|
|
}
|
|
}
|
|
if found == nil {
|
|
// Bound to something answered on this machine rather than from the mesh. Nothing
|
|
// to write: the answer is here, and a file saying "it is on this node" would be
|
|
// a fact nobody needs and one more thing to keep true.
|
|
continue
|
|
}
|
|
file, err := boundFile(*found, m.Binds[to])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
first = append(first, file)
|
|
}
|
|
for _, to := range sortedKeys(m.Receives) {
|
|
file, err := receivedFile(to, m.Receives[to], given[to])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
first = append(first, file)
|
|
}
|
|
if m.Computed != "" {
|
|
generator, known := with.Generators[m.Computed]
|
|
if !known {
|
|
return nil, fmt.Errorf(
|
|
"%s says its resources are computed by %q, and this control plane has no %q",
|
|
m.Module, m.Computed, m.Computed)
|
|
}
|
|
generated, part, err := generator.Resources(r.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !part {
|
|
// Assigned, and not yet part of what this generates. Nothing to put on the
|
|
// machine, which is different from an error: a node given the network module
|
|
// before it has an address is in exactly that state, briefly.
|
|
continue
|
|
}
|
|
resources = generated
|
|
}
|
|
|
|
// Now, and not before: a module whose resources are computed replaces them wholesale, and
|
|
// merging earlier would throw away the files it still needs.
|
|
resources = append(append([]map[string]any{}, first...), resources...)
|
|
|
|
for _, unsettled := range resources {
|
|
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
copied := map[string]any{}
|
|
for k, v := range resource {
|
|
copied[k] = v
|
|
}
|
|
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
|
|
// A service saying what it reflects names resources within its own module, so those
|
|
// are prefixed too or they would point at nothing.
|
|
if reflects, ok := resource["restart-on"].([]any); ok {
|
|
var renamed []any
|
|
for _, id := range reflects {
|
|
renamed = append(renamed, m.Module+"."+fmt.Sprint(id))
|
|
}
|
|
copied["restart-on"] = renamed
|
|
}
|
|
out = append(out, copied)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Contribution is one module telling the answer to a requirement what it needs from it.
|
|
type Contribution struct {
|
|
// From is the module that said it, so the provider and a person reading the file can tell
|
|
// which route belongs to what.
|
|
From string `json:"from"`
|
|
// Node is the machine it said it from, empty when that is this one.
|
|
//
|
|
// A provision answered from anywhere in the mesh has consumers on other machines, and the
|
|
// provider has to know who they are — a database told to create a password and not who for
|
|
// cannot do anything with it. Contributions were node-local until this, which meant the one
|
|
// case that most needed them was the one they did not reach.
|
|
Node string `json:"node,omitempty"`
|
|
// Secret is the file on this machine holding that consumer's credential, sealed to it.
|
|
//
|
|
// Named rather than carried, for the same reason the private network's key is: the mesh
|
|
// discarded the value and could not put it here if it wanted to. What is here is where to
|
|
// find it.
|
|
Secret string `json:"secret,omitempty"`
|
|
// Values are the module's own, with settings applied. What the keys mean is agreed by the
|
|
// requirement's name — everything providing `reverse-proxy` understands the same shape, which
|
|
// is what makes swapping one for another cost nothing.
|
|
Values map[string]any `json:"values"`
|
|
}
|
|
|
|
// grantPath is where one consumer's sealed credential lands on the providing machine.
|
|
//
|
|
// Suffixed, so the directory can also hold whatever the module writing it keeps there and so a
|
|
// node named like something else in that directory cannot collide with it.
|
|
func grantPath(directory, consumer string) string {
|
|
return strings.TrimRight(directory, "/") + "/" + consumer + ".secret"
|
|
}
|
|
|
|
// contributions collects what every module in this set contributes, by requirement.
|
|
//
|
|
// Ordered by contributing module, because the result becomes a file on a machine and a file whose
|
|
// lines move about is a file that looks changed when nothing changed.
|
|
func (r Resolution) contributions(settings SettingsBy, grants []Grant,
|
|
directories map[string]string) (map[string][]Contribution, error) {
|
|
out := map[string][]Contribution{}
|
|
modules := append([]Manifest{}, r.Modules...)
|
|
sort.Slice(modules, func(i, j int) bool { return modules[i].Module < modules[j].Module })
|
|
|
|
// What consumers on other machines asked for. Merged in with this machine's own, because from
|
|
// the provider's side they are the same thing — somebody wanting something — and a provider
|
|
// that had to read two lists would be a provider that reads one of them.
|
|
sorted := append([]Grant{}, grants...)
|
|
sort.Slice(sorted, func(i, j int) bool {
|
|
if sorted[i].Provision != sorted[j].Provision {
|
|
return sorted[i].Provision < sorted[j].Provision
|
|
}
|
|
return sorted[i].Consumer < sorted[j].Consumer
|
|
})
|
|
for _, g := range sorted {
|
|
if g.From == "" {
|
|
// As above: nothing on that machine asks for this any more, so the provider is not
|
|
// told about it and withdraws the login on its next pass.
|
|
continue
|
|
}
|
|
out[g.Provision] = append(out[g.Provision], Contribution{
|
|
From: g.From, Node: g.Consumer, Values: g.Values,
|
|
Secret: grantPath(directories[g.Provision], g.Consumer),
|
|
})
|
|
}
|
|
for _, m := range modules {
|
|
for _, to := range sortedKeys(m.Contributes) {
|
|
// Settings reach a contribution the same way they reach a file. A route's hostname is
|
|
// exactly the kind of thing that differs between one mesh and the next, and a module
|
|
// that could not have it set would have to be edited to be reused.
|
|
values, err := settle(m.Contributes[to], settings[m.Module], nil,
|
|
m.Module+" contributing to "+to)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
|
|
}
|
|
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// receivedFile is the file a provider is given its consumers' contributions in.
|
|
func receivedFile(requirement, path string, given []Contribution) (map[string]any, error) {
|
|
if given == nil {
|
|
// Nobody contributed. The file is still written, empty, rather than left absent: a
|
|
// provider that finds no file cannot tell "nothing asked for me" from "the mesh never
|
|
// wrote it", and the two want completely different responses.
|
|
given = []Contribution{}
|
|
}
|
|
// The note goes *inside* the document, not above it. The first version wrote a `//` header
|
|
// and produced a file that says "do not edit" to a person and fails to parse for the program
|
|
// meant to read it — which is the whole audience.
|
|
body, err := json.MarshalIndent(map[string]any{
|
|
"contributions": 1,
|
|
"requirement": requirement,
|
|
"generated": "by the mesh — do not edit; replaced whenever a module contributing to " +
|
|
requirement + " arrives or leaves",
|
|
"given": given,
|
|
}, "", " ")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{
|
|
"id": ReceivedID(requirement), "type": "file", "path": path, "mode": "0644",
|
|
"content": string(body) + "\n",
|
|
}, nil
|
|
}
|
|
|
|
// sortedKeys is map iteration made repeatable, which everything written to a machine needs.
|
|
func sortedKeys[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// boundFile is what a module is told about something it requires from another machine.
|
|
//
|
|
// Where it is and what the providing module said about using it. **No credential**, and the file
|
|
// says so rather than leaving a reader to wonder whether one was meant to be there — a missing
|
|
// field looks like a bug, and a stated absence looks like a boundary.
|
|
func boundFile(n Needed, path string) (map[string]any, error) {
|
|
body, err := json.MarshalIndent(map[string]any{
|
|
"binding": 1,
|
|
"provision": n.Name,
|
|
"from": n.From,
|
|
"at": n.At,
|
|
"serves": n.Serves,
|
|
"generated": "by the mesh — do not edit; replaced whenever this changes. " +
|
|
"It carries no credential: the mesh has no way to issue one yet",
|
|
}, "", " ")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{
|
|
"id": BoundID(n.Name), "type": "file", "path": path, "mode": "0644",
|
|
"content": string(body) + "\n",
|
|
}, nil
|
|
}
|
|
|
|
// ContributionsTo is what this node's set asked of one requirement, settled.
|
|
//
|
|
// Exported because a provider's grants are assembled from its consumers' resolutions, one machine
|
|
// at a time, and the alternative was for the control plane to reimplement settling.
|
|
func (r Resolution) ContributionsTo(requirement string, settings SettingsBy) (
|
|
string, map[string]any, error) {
|
|
all, err := r.contributions(settings, nil, nil)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
given := all[requirement]
|
|
if len(given) == 0 {
|
|
return "", nil, nil
|
|
}
|
|
if len(given) > 1 {
|
|
// Two modules on one machine wanting the same provision would share one credential, and
|
|
// the provider would be told to create one thing under two names. Refused rather than
|
|
// resolved by picking, which is the rule everywhere else here.
|
|
var who []string
|
|
for _, g := range given {
|
|
who = append(who, g.From)
|
|
}
|
|
sort.Strings(who)
|
|
return "", nil, fmt.Errorf(
|
|
"%s has %d modules asking for %q and they would share one credential: %s",
|
|
r.Node, len(given), requirement, strings.Join(who, ", "))
|
|
}
|
|
return given[0].From, given[0].Values, nil
|
|
}
|