novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

The roster is the machines: each node's internal domain covers its routes (hq ADR 0191) #238

Merged
jschoubben merged 2 commits from fix/the-mesh-publishes-the-names-it-composed into main 2026-10-03 19:12:24 +00:00
Conversation 0 Commits 2 Files Changed 8
+41 -392
7 changed files with 41 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 11e4bc0ba1 - Show all commits
cmd/mesh-controller/network_test.go
+26
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"os"
"reflect"
"strings"
"testing"
@@ -303,3 +304,28 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
}
}
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(with.Names, with.Machines) {
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
}
}
}
cmd/mesh-controller/plan.go
+7 -82
View File
@@ -645,22 +645,17 @@ func renderingFor(ctx context.Context, open *stores, node string,
}
}
// And every route's internal name → the node that serves it, alongside the `<node>.internal`
// names above (novox/hq ADR 0066, narrowed by ADR 0191). A route's public name is not among
// them: the mesh gives no private answer for a name public DNS answers.
// Kept apart from the machines, because a fact about the machines must not be handed the names
// the mesh merely serves (novox/hq 04-ISSUES/111).
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
// answer for any of them. The roster once carried every routed name, public ones included, and a
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
machines := make(map[string]string, len(names))
for name, at := range names {
machines[name] = at
}
routes, err := routeNamesInTheMesh(ctx, open)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
for name, at := range routes {
names[name] = at
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
@@ -739,76 +734,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
}, record, nil
}
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
// ADR 0066).
//
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
// composed on the consumer's node (from its label and that node's public domain) and served by the
// node answering the consumer's route requirement; this gathers both.
//
// It reads route names off resolutions rather than a table because there is no table: a route is a
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
// routed name only because it carried a label the mesh composed, never because the mesh knows what
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
// the rest their names.
//
// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
// every machine at once, that its names do not exist — and since the roster is part of every
// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
// 151). So every failure here says which machine and which read, because the alternative is a
// mesh-wide refusal with nothing named in it.
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
}
address := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
address[p.Name] = p.Address
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
}
// Every machine's resolution first, then the names across them at once: which node serves a
// name is a question about the graph — the consumer on one machine, the provider on another —
// and answered wrongly by looking at one contribution at a time (novox/hq issue 178).
plans := map[string]catalogue.Resolution{}
settings := map[string]catalogue.SettingsBy{}
for _, n := range nodes {
plan, layers, err := planFor(ctx, open, n.Name)
switch {
case unresolvable(err):
// Their set does not compose, so they serve no names. Passed over, so one machine's
// broken set does not cost the rest theirs.
continue
case err != nil:
// The mesh could not be asked. Returning the roster without this machine's names would
// state that they do not exist — to every machine, and indistinguishably from the
// operator having withdrawn them (novox/hq 04-ISSUES/152).
return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
}
plans[n.Name], settings[n.Name] = plan, layers
}
served, err := catalogue.NamesServed(plans, settings)
if err != nil {
return nil, err
}
out := map[string]string{}
for name, node := range served {
if at := address[node]; at != "" {
out[name] = at
}
}
return out, nil
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
cmd/mesh-controller/roster_failure_test.go
+1 -54
View File
@@ -2,13 +2,12 @@ package main
import (
"context"
"strings"
"testing"
)
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
// things, and only the first may be passed over when something is gathered across every machine
// (novox/hq 04-ISSUES/152). These pin that distinction where the three gatherers rely on it.
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
open := aMesh(t)
@@ -45,55 +44,3 @@ func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
}
}
func TestOneIncoherentNodeDoesNotCostTheRestTheirNames(t *testing.T) {
open := aMesh(t)
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{one.Module, two.Module} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
// laptop cannot compose. That is laptop's problem and nobody else's: the roster is still
// answerable, and anchor keeps whatever it serves.
if _, err := routeNamesInTheMesh(t.Context(), open); err != nil {
t.Fatalf("one node's broken set cost the whole mesh its roster: %v", err)
}
}
func TestARosterIsNeverReturnedWithNamesItCouldNotRead(t *testing.T) {
open := aMesh(t)
stopped, cancel := context.WithCancel(t.Context())
cancel()
names, err := routeNamesInTheMesh(stopped, open)
if err == nil {
t.Fatalf("a roster was composed from a store that could not be read: %v", names)
}
// The failure must be raised, not turned into an absence. A roster missing a machine's names
// is indistinguishable, on every machine that receives it, from the operator withdrawing them —
// and because the roster is part of every container's identity, it replaces all of them.
if names != nil {
t.Fatalf("a partial roster was returned beside the error: %v", names)
}
}
// Kept so the reason survives the next person reading it: the message the gatherer raises must say
// which machine could not be read, or the operator is left with a mesh-wide failure and no name.
func TestTheRaisedFailureNamesTheMachineItCouldNotRead(t *testing.T) {
open := aMesh(t)
stopped, cancel := context.WithCancel(t.Context())
cancel()
_, err := routeNamesInTheMesh(stopped, open)
if err == nil {
t.Fatal("no failure was raised")
}
if !strings.Contains(err.Error(), "cannot be read") {
t.Fatalf("the failure does not say the mesh could not be read: %v", err)
}
}
internal/catalogue/names_served.go
-133
View File
@@ -1,133 +0,0 @@
package catalogue
import (
"sort"
"strings"
)
// Which machine serves each routed name (novox/hq ADR 0066, issue 178).
//
// A routed name is a label the mesh composed for a consumer's endpoint, and it is *served* by the
// provider that answers requests for it — the proxy the consumer's route reaches. The same name is
// composed into every labelled contribution the consumer makes, because a provider that must know
// the consumer's public name (an identity provider composing a redirect) is told it the same way
// (04-ISSUES/122). Attributing the name to whichever of those providers a map happened to yield
// last sent a public name to the identity provider's machine on one plan and to the proxy's on the
// next (forge issue 227), and the whole names region flipped with it.
//
// **The terminus serves the name.** Among the providers a name reaches, the one that serves it is
// the one that is not itself routed: a provider that contributes a labelled name of its own to some
// requirement is published through another provider, and is a consumer of names, not their end.
// Name-agnostic — nothing here knows what "route" means — and structural: it reads the graph the
// modules declared. Deterministic: names, requirements and nodes are walked in order, so two
// plans of one mesh yield one region.
// **The name published is the one the mesh composed for itself** (novox/hq ADR 0191). A route carries
// two: `name`, composed from the node's public domain, and `internal-name`, composed from the mesh's
// own domain under the serving node (ADR 0151). Only the second is the mesh's to answer. The public
// one is the operator's, answered by public DNS — publishing it here gave every machine's resolver a
// private answer for a public name, and a resolver that also serves a LAN handed it to a phone that
// could not reach it. Which is which is known from where each was composed, not read off its spelling.
//
// NamesServed is every routed internal name across the mesh and the node that serves it, from every
// node's resolution and settings. A name several termini claim goes to the first node in name order, so
// the answer is stable; a name nothing terminal claims is left out.
func NamesServed(plans map[string]Resolution, settings map[string]SettingsBy) (map[string]string, error) {
nodes := make([]string, 0, len(plans))
for n := range plans {
nodes = append(nodes, n)
}
sort.Strings(nodes)
out := map[string]string{}
for _, node := range nodes {
plan := plans[node]
all, err := plan.contributions(settings[node], nil, nil)
if err != nil {
return nil, err
}
requirements := make([]string, 0, len(all))
for to := range all {
requirements = append(requirements, to)
}
sort.Strings(requirements)
for _, to := range requirements {
for _, given := range all[to] {
if given.Node != "" {
// Said from another machine; that machine's own resolution carries it.
continue
}
// A routed name, and only that: a contribution the mesh composed a name for from a
// label it was given. A grant that happens to carry a `name` of its own — a database
// name — carries no label and is left alone.
if _, labelled := given.Values["label"]; !labelled {
continue
}
// A route that asked for no internal name — its reach is public only — has none to
// publish, and its public name is not the mesh's to answer.
name, _ := given.Values["internal-name"].(string)
if name == "" {
continue
}
serving := servingNodeOf(plan, to, given.From, node)
if !servesNames(plans[serving], to) {
continue
}
name = strings.ToLower(name)
if held, taken := out[name]; !taken || serving < held {
out[name] = serving
}
}
}
}
return out, nil
}
// servingNodeOf is the node answering one consumer's requirement: whoever the plan needs it from,
// or this same node when the provider is beside the consumer.
func servingNodeOf(plan Resolution, requirement, consumer, self string) string {
for _, need := range plan.Needs {
if need.Name == requirement && need.For == consumer && need.From != "" {
return need.From
}
}
return self
}
// servesNames says whether the module providing a requirement on a node is a terminus: it is not
// itself published under a labelled name through some other provider. A node whose plan is not
// known (it did not resolve) serves nothing.
func servesNames(plan Resolution, requirement string) bool {
for _, m := range plan.Modules {
if !offers(m, requirement) {
continue
}
return !contributesALabel(m)
}
return false
}
func offers(m Manifest, requirement string) bool {
for _, o := range m.Offers() {
if o == requirement {
return true
}
}
return false
}
func contributesALabel(m Manifest) bool {
for _, values := range m.Contributes {
if _, labelled := values["label"]; labelled {
return true
}
}
for _, locals := range m.ContributesMany {
for _, values := range locals {
if _, labelled := values["label"]; labelled {
return true
}
}
}
return false
}
internal/catalogue/names_served_test.go
-115
View File
@@ -1,115 +0,0 @@
package catalogue
import (
"testing"
)
// The mesh of forge issue 227 (novox/hq issue 178): a dashboard on the home server contributes its
// label to the route its proxy serves AND to the identity provider on the control node, which must
// know the dashboard's public name to compose a redirect. Both contributions carry the composed
// name; only the proxy serves it.
func twoNodesOneName(t *testing.T) (map[string]Resolution, map[string]SettingsBy) {
t.Helper()
catalogue := shelf(
Manifest{Module: "route-adapter", Version: "1", Provides: Offers("route"),
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/adapter/mesh.json"}},
Manifest{Module: "route-proxy", Version: "1", Provides: Offers("route"),
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/proxy/mesh.json"}},
Manifest{Module: "keycloak", Version: "1", Provides: FromAnywhere("oidc-client"),
Serves: map[string]map[string]any{"oidc-client": {"token-path": "/token"}},
Receives: map[string]string{"oidc-client": "/etc/keycloak/clients.json"},
Listens: []Listening{{Port: 8080, From: FromMesh, Why: "the login page"}},
// Published through the proxy itself: the identity provider is routed, not a router.
Contributes: map[string]map[string]any{"route": {"label": "login", "endpoint": "web", "port": 8080}}},
Manifest{Module: "grafana", Version: "1",
Listens: []Listening{{Port: 3000, From: FromMesh, Why: "dashboards"}},
Contributes: map[string]map[string]any{
"route": {"label": "grafana", "endpoint": "web", "port": 3000},
"oidc-client": {"label": "grafana", "endpoint": "web", "port": 3000, "callback": "/login"},
}},
)
home := withDomain("home.example")
home.Name, home.At = "home-server", "home-server.internal"
control := withDomain("control.example")
control.Name, control.At = "anchor", "anchor.internal"
onHome, err := Resolve(catalogue, []string{"grafana", "route-adapter"}, home, World{
Offered: map[string][]Provider{"oidc-client": {{Node: "anchor", At: "anchor.internal", Module: "keycloak"}}},
})
if err != nil {
t.Fatal(err)
}
onControl, err := Resolve(catalogue, []string{"keycloak", "route-proxy"}, control, World{})
if err != nil {
t.Fatal(err)
}
return map[string]Resolution{"home-server": onHome, "anchor": onControl},
map[string]SettingsBy{"home-server": {}, "anchor": {}}
}
func TestANameResolvesToTheNodeWhoseProxyServesIt(t *testing.T) {
plans, settings := twoNodesOneName(t)
// Many times, because the fault was map order: one plan said one node, the next the other.
for i := 0; i < 25; i++ {
served, err := NamesServed(plans, settings)
if err != nil {
t.Fatal(err)
}
if served["grafana.home-server.internal"] != "home-server" {
t.Fatalf("run %d: the dashboard's name is served by %q, and its proxy is on the home server: %v",
i, served["grafana.home-server.internal"], served)
}
if served["login.anchor.internal"] != "anchor" {
t.Fatalf("run %d: the identity provider's own name is served by its proxy on the control node: %v", i, served)
}
if _, leaked := served["grafana.anchor.internal"]; leaked {
t.Fatalf("a name composed for the identity provider's benefit is not one it serves: %v", served)
}
}
}
// A module that is routed several times names each route (ADR 0094's sibling for contributes);
// every one of them is a name the mesh must resolve, and none reached the names region before.
func TestEveryRouteOfAModuleWithSeveralIsANameServed(t *testing.T) {
catalogue := shelf(
Manifest{Module: "route-proxy", Version: "1", Provides: Offers("route"),
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/proxy/mesh.json"}},
Manifest{Module: "photos", Version: "1",
Listens: []Listening{{Port: 8102, From: FromMesh, Why: "web"}, {Port: 9102, From: FromMesh, Why: "api"}},
ContributesMany: map[string]map[string]map[string]any{"route": {
"site": {"label": "photos", "endpoint": "web", "port": 8102},
"api": {"label": "photos-api", "endpoint": "api", "port": 9102},
}}},
)
node := withDomain("control.example")
node.Name, node.At = "anchor", "anchor.internal"
plan, err := Resolve(catalogue, []string{"photos", "route-proxy"}, node, World{})
if err != nil {
t.Fatal(err)
}
served, err := NamesServed(map[string]Resolution{"anchor": plan}, map[string]SettingsBy{"anchor": {}})
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"photos.anchor.internal", "photos-api.anchor.internal"} {
if served[name] != "anchor" {
t.Fatalf("%s is not served by its proxy: %v", name, served)
}
}
}
// A route's public name is the operator's and answered by public DNS; the mesh publishes only the
// internal name it composed for the same route (novox/hq ADR 0191) — told apart by where each was
// composed, never by how it is spelled.
func TestAPublicNameIsNeverAMeshName(t *testing.T) {
plans, settings := twoNodesOneName(t)
served, err := NamesServed(plans, settings)
if err != nil {
t.Fatal(err)
}
for _, public := range []string{"grafana.home.example", "login.control.example"} {
if node, published := served[public]; published {
t.Fatalf("the public name %s is published at %s; public DNS answers it: %v", public, node, served)
}
}
}
internal/catalogue/roster.go
+4 -4
View File
@@ -28,10 +28,10 @@ import (
// A RosterFile is a file the mesh renders from the roster of machines, in the format the module
// gives as a Go text/template. The template sees a rosterView: `.Node` (this machine's bare name),
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names`, every
// name the mesh serves, and `.Machines`, only the nodes of the mesh. Which set a template ranges is
// how the hq issue 111 distinction is drawn: a container's hosts wants every name; a resolver told
// the suffix is its own wants only the machines.
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names` and
// `.Machines`. Both are the nodes of the mesh: routed names were once in `.Names` too, and are not
// since every route became a name under its node's internal domain (novox/hq ADR 0191) — the hq
// issue 111 distinction is kept as two fields so the templates that range either keep rendering.
type RosterFile struct {
// Path is where on the machine the rendered file goes. Absolute, or it is refused here rather
// than discovered as a daemon that reads nothing.
internal/overlay/generator.go
+3 -4
View File
@@ -169,10 +169,9 @@ func (g *Generator) Graph() Graph { return g.graph }
//
// - No floor: no header, no localhost, no `127.0.1.1` — those are the machine's, above the region.
// - A machine's own line is marked, and its mesh name resolves to its mesh address, not loopback.
// - `.Names` is every name the mesh serves (issue 111), so anything on the machine reaching a
// routed name through its resolver finds the machine serving it; machines with no address yet
// are already left out of the set. A routed name is one alias, itself — a machine has a bare
// name beside its full one, a routed name has nothing beside it (issue 157).
// - `.Names` is the machines (novox/hq ADR 0191): a route's internal name is under its node's
// internal domain and the resolver answers it by wildcard, and a public name is public DNS's.
// Machines with no address yet are already left out of the set.
const hostsTemplate = "# The mesh's names. This region is replaced whenever a machine joins or leaves.\n" +
"{{range .Names}}{{.Address}}\t{{.FQDN}}{{if ne .Name .FQDN}}\t{{.Name}}{{end}}{{if eq .Name $.Node}}\t# this machine{{end}}\n{{end}}"