From bff893f8af9b0d65f45cc83409e3e0661ca532ad Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 31 Aug 2026 12:21:58 +0200 Subject: [PATCH] Resolver modules: one that serves, and two ways of deciding what a machine asks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three manifests and the rule that keeps them apart. Serving and asking are genuinely different roles, and systemd-resolved can only do the second — it cannot answer a wildcard, it routes the mesh's suffix to something that can. A module that treated them as one role could not work, which is the mistake worth naming rather than discovering. So `the-dns-port` and `the-resolver-configuration` are two claims. A machine gets one of each, and two of either is refused by the mesh rather than fought over on the machine — which is what ADR 0009's table meant by listing resolvers beside the seat and pid 1. That table names the resource `/etc/resolv.conf`, which is what it is; a claim is a name in the catalogue's own form, and the catalogue refuses the path as one. Neither module knows anything about the machine it is on, which is what lets them be static manifests: they name `mesh0` and `127.0.0.54`, both chosen by the mesh, rather than an address only that machine has. Not 127.0.0.1 and not 127.0.0.53 — taking either would be a module claiming something it did not say it claims. A service can now reflect a file another module put on the machine, written `.`. The resolver has to restart when the mesh rewrites the names; without it, it would serve the names it started with for ever, with every machine that joined afterwards unreachable and every check passing. --- examples/modules/README.md | 42 ++++++ examples/modules/dnsmasq.json | 24 ++++ examples/modules/modules_test.go | 157 +++++++++++++++++++++++ examples/modules/resolv-conf.json | 12 ++ examples/modules/resolved-split-dns.json | 18 +++ internal/catalogue/declaration.go | 14 +- internal/catalogue/filtering_test.go | 44 +++++++ 7 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 examples/modules/README.md create mode 100644 examples/modules/dnsmasq.json create mode 100644 examples/modules/modules_test.go create mode 100644 examples/modules/resolv-conf.json create mode 100644 examples/modules/resolved-split-dns.json diff --git a/examples/modules/README.md b/examples/modules/README.md new file mode 100644 index 0000000..5962d21 --- /dev/null +++ b/examples/modules/README.md @@ -0,0 +1,42 @@ +# Example modules + +Manifests, not programs. They are here because the contract is easier to read as something that +works than as a description of something that would. + +**Third-party software runs *on* the mesh, not *of* it** (novox/hq ADR 0001). dnsmasq is not the +mesh's, and neither is systemd-resolved — what is the mesh's is the fact only it can know, which +is which machines exist and where they are. So the mesh writes that to a file and these read it. + +## Resolving a service named under a machine + +`postgres.novox.internal`, `plex.ace.internal`. The first label is the service and the rest is the +node, so **anything under a node's name must resolve to that node** and a proxy there routes by +the name it was asked for. That routing is a separate concern and stays separate. + +Two roles, and they are genuinely different things: + +| | claims | | +|---|---|---| +| **serving** | `the-dns-port` | answers the wildcards — `dnsmasq.json` | +| **asking** | `the-resolver-configuration` | decides what the machine asks — `resolved-split-dns.json`, `resolv-conf.json` | + +**systemd-resolved cannot serve a wildcard**, so it is only ever an *asking* module: it routes the +mesh's suffix to something that can. Treating the two roles as one would produce a module that +cannot work, which is the mistake worth naming. + +Assign one of each. Two of either is refused by the mesh rather than fought over on the machine: + +> `resolved-split-dns and resolv-conf both claim "the-resolver-configuration", and only one thing +> may hold it per node` + +ADR 0009's table names that resource `/etc/resolv.conf`, which is what it *is*, the way it writes +*the seat*. A claim is a name in the catalogue's own form, so it is written as one. + +## Why neither needs to know the machine's address + +Both would ordinarily need it — a resolver must bind somewhere, and a stub must be pointed +somewhere — and a static manifest cannot know it. + +Neither does, because both name things **the mesh itself named**: the private network's interface +is `mesh0` on every machine, and the address a resolver listens on for the machine's own use is +`127.0.0.54` on every machine. A name the mesh chose is a name a manifest can use. diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json new file mode 100644 index 0000000..15d2059 --- /dev/null +++ b/examples/modules/dnsmasq.json @@ -0,0 +1,24 @@ +{ + "module": "dnsmasq", + "version": "1", + + "requires": ["resolver-data"], + "provides": ["wildcard-resolution"], + "claims": [{"name": "the-dns-port", "scope": "node"}], + + "listens": [ + {"port": 53, "protocol": "udp", "from": "mesh", + "why": "names under every machine in this mesh, for this machine and what it runs"} + ], + + "resources": [ + {"id": "package", "type": "package", "package": "dnsmasq"}, + + {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it — including a container\n# on this machine — can ask.\n# 127.0.0.54 this machine's own use. Not 127.0.0.1 and not 127.0.0.53:\n# the first is where everything else expects a resolver, and the\n# second is systemd-resolved's. Taking either would be this\n# module claiming something it did not say it claims.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.54\n\n# It answers for the mesh and forwards nothing it was not asked about. Names\n# outside the mesh are somebody else's business, and a resolver that answered\n# them would be this module taking over more than it claims.\ndomain-needed\nbogus-priv\n"}, + + {"id": "service", "type": "service", "unit": "dnsmasq.service", + "state": "running", "boot": "enabled", + "restart-on": ["config", "mesh-resolver.nodes"]} + ] +} diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go new file mode 100644 index 0000000..530e862 --- /dev/null +++ b/examples/modules/modules_test.go @@ -0,0 +1,157 @@ +package modules + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/overlay" +) + +// The examples are manifests, so the thing to check is that the catalogue accepts them. +// +// A manifest that only ever appears in a document is a manifest nobody has run through the parser, +// and the parser refuses unknown keys — so a typo here would be discovered by whoever first tried +// to use one, which is the opposite of what an example is for. +func read(t *testing.T, name string) catalogue.Manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join(".", name)) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("%s is not a manifest this mesh accepts: %v", name, err) + } + return m +} + +func TestEveryExampleIsAManifestTheMeshAccepts(t *testing.T) { + found, err := filepath.Glob("*.json") + if err != nil { + t.Fatal(err) + } + if len(found) == 0 { + t.Fatal("no examples, so this test proves nothing") + } + for _, name := range found { + read(t, name) + } +} + +// The serving module reads what the mesh writes, and restarts when the mesh rewrites it. +// +// Without the second it would serve the names it started with for ever — every machine that +// joined afterwards unreachable by name, and every check passing. +func TestTheResolverReadsTheMeshsNamesAndFollowsThem(t *testing.T) { + m := read(t, "dnsmasq.json") + + var config, service map[string]any + for _, r := range m.Resources { + switch r["id"] { + case "config": + config = r + case "service": + service = r + } + } + if config == nil || service == nil { + t.Fatal("the module has no configuration or no service") + } + if !strings.Contains(config["content"].(string), overlay.ResolverPath) { + t.Fatalf("it does not read what the mesh writes at %s", overlay.ResolverPath) + } + + var follows bool + for _, id := range service["restart-on"].([]any) { + if id.(string) == overlay.Resolver+".nodes" { + follows = true + } + } + if !follows { + t.Fatalf("it does not restart when the mesh rewrites the names: %v", service["restart-on"]) + } +} + +// It binds names the mesh chose, so it needs to know nothing about the machine it is on. +// +// That is the whole reason these can be static manifests: a resolver must bind somewhere and a +// stub must be pointed somewhere, and neither address is knowable in advance — unless the mesh +// named it. +func TestTheResolverNeedsToKnowNothingAboutItsMachine(t *testing.T) { + config := read(t, "dnsmasq.json").Resources[1]["content"].(string) + // The directive, not the word: the comment above it names the interface too, so a plain + // Contains passes whatever the module actually binds. It did. + if !strings.Contains(config, "interface="+overlay.Interface+"\n") { + t.Fatalf("it does not bind the private network's interface %q:\n%s", + overlay.Interface, config) + } + if !strings.Contains(config, "listen-address=127.0.0.54\n") { + t.Fatalf("it does not answer on the address the asking modules point at:\n%s", config) + } + // Not the two addresses that belong to something else. + for _, taken := range []string{"listen-address=127.0.0.1", "listen-address=127.0.0.53"} { + if strings.Contains(config, taken) { + t.Fatalf("it takes %q, which belongs to something it did not claim:\n%s", taken, config) + } + } +} + +// The two ways of deciding what a machine asks claim the same thing, so the mesh refuses the pair. +func TestTwoWaysOfOwningTheResolverCannotBothBeAssigned(t *testing.T) { + shelf := map[string]catalogue.Manifest{} + for _, name := range []string{"resolved-split-dns.json", "resolv-conf.json", "dnsmasq.json"} { + m := read(t, name) + shelf[m.Module] = m + } + // Something has to answer `wildcard-resolution`, or they are refused for that instead and the + // test would pass without ever reaching the claim. + shelf["dnsmasq"] = read(t, "dnsmasq.json") + + _, err := catalogue.Resolve(shelf, + []string{"dnsmasq", "resolved-split-dns", "resolv-conf"}, + catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, + catalogue.World{Unchecked: true}) + if err == nil { + t.Fatal("both ways of owning the resolver were assigned to one machine") + } + said := err.Error() + if !strings.Contains(said, "the-resolver-configuration") { + t.Fatalf("the refusal does not name what they both want: %v", said) + } +} + +// And the two roles are not the same claim: a machine runs one resolver AND one thing deciding +// what it asks, so serving and asking must be assignable together. +func TestServingAndAskingAreAssignableTogether(t *testing.T) { + shelf := map[string]catalogue.Manifest{} + for _, name := range []string{"dnsmasq.json", "resolved-split-dns.json"} { + m := read(t, name) + shelf[m.Module] = m + } + if _, err := catalogue.Resolve(shelf, + []string{"dnsmasq", "resolved-split-dns"}, + catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, + catalogue.World{Unchecked: true}); err != nil { + t.Fatalf("a resolver and the thing pointing at it cannot both be assigned: %v", err) + } +} + +// The examples are JSON a person edits, so a stray comma is worth catching here rather than on a +// machine. +func TestTheExamplesAreWellFormed(t *testing.T) { + found, _ := filepath.Glob("*.json") + for _, name := range found { + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + var any map[string]any + if err := json.Unmarshal(raw, &any); err != nil { + t.Fatalf("%s is not JSON: %v", name, err) + } + } +} diff --git a/examples/modules/resolv-conf.json b/examples/modules/resolv-conf.json new file mode 100644 index 0000000..5631573 --- /dev/null +++ b/examples/modules/resolv-conf.json @@ -0,0 +1,12 @@ +{ + "module": "resolv-conf", + "version": "1", + + "requires": ["wildcard-resolution"], + "claims": [{"name": "the-resolver-configuration", "scope": "node"}], + + "resources": [ + {"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644", + "content": "# Managed by the mesh.\n#\n# For a machine where nothing else owns this file. On one where systemd-resolved\n# or NetworkManager does, assign that module instead — this one and those claim\n# the same thing, so the mesh refuses the pair rather than letting them take\n# turns overwriting each other, which is the failure this claim exists to stop.\n#\n# The mesh's resolver first, because it answers only the mesh's names and\n# forwards nothing: a query it does not recognise falls through to the next\n# line rather than being answered wrongly.\nnameserver 127.0.0.54\n\n# And what this machine used before. Replace this line with the resolver this\n# machine should use for everything that is not the mesh — it is not the mesh's\n# to choose, and a public one written here by default would send every query\n# this machine makes somewhere nobody agreed to.\nnameserver 127.0.0.53\n"} + ] +} diff --git a/examples/modules/resolved-split-dns.json b/examples/modules/resolved-split-dns.json new file mode 100644 index 0000000..cbe99ea --- /dev/null +++ b/examples/modules/resolved-split-dns.json @@ -0,0 +1,18 @@ +{ + "module": "resolved-split-dns", + "version": "1", + + "requires": ["wildcard-resolution"], + "claims": [{"name": "the-resolver-configuration", "scope": "node"}], + + "resources": [ + {"id": "drop-in", "type": "directory", "path": "/etc/systemd/resolved.conf.d", "mode": "0755"}, + + {"id": "route", "type": "file", + "path": "/etc/systemd/resolved.conf.d/mesh.conf", "mode": "0644", + "content": "# Managed by the mesh.\n#\n# **Only the mesh's names.** The tilde makes this a routing domain rather than a\n# search domain: queries under it go to the resolver below, and everything else\n# keeps going wherever this machine already sent it. A resolver that took over\n# all of DNS would be this module claiming the machine's whole network, which\n# is not what it says it claims.\n#\n# 127.0.0.54 is where the mesh's resolver answers on every machine — a name the\n# mesh chose, so this file needs to know nothing about this particular one.\n[Resolve]\nDNS=127.0.0.54\nDomains=~internal\n"}, + + {"id": "resolved", "type": "service", "unit": "systemd-resolved.service", + "state": "running", "boot": "enabled", "restart-on": ["route"]} + ] +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 6cefc81..1b62383 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -314,10 +314,22 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those // are prefixed too or they would point at nothing. + // + // **Unless it already names one.** A module may reflect a file another module put on + // the machine — the case this exists for is a resolver restarting when the mesh + // rewrites the names, which are computed by the mesh and belong to its module, not to + // the daemon's. Written `.`, and a dot is what marks it as already + // answered: prefixing it again would point at nothing, silently, and the daemon would + // serve the old names for ever while everything reported success. if reflects, ok := resource["restart-on"].([]any); ok { var renamed []any for _, id := range reflects { - renamed = append(renamed, m.Module+"."+fmt.Sprint(id)) + named := fmt.Sprint(id) + if strings.Contains(named, ".") { + renamed = append(renamed, named) + continue + } + renamed = append(renamed, m.Module+"."+named) } copied["restart-on"] = renamed } diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index b029ca2..408f376 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -458,3 +458,47 @@ func TestAGeneratorThatOpensNothingNeedsNoMethod(t *testing.T) { t.Fatalf("a generator that says nothing about ports opened one: %+v", rules) } } + +// A module may reflect a file another module put on the machine. +// +// The case this exists for: a resolver restarting when the mesh rewrites the names. Those are +// computed by the mesh and belong to its module, not to the daemon's — so without this the +// daemon would serve the names it started with for ever, while every check reported success. +func TestAServiceCanReflectAnotherModulesFile(t *testing.T) { + out, err := Resolution{Node: "anchor", Modules: []Manifest{{ + Module: "dnsmasq", + Resources: []map[string]any{ + {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", + "content": "x", "mode": "0644"}, + {"id": "run", "type": "service", "unit": "dnsmasq.service", "state": "running", + "restart-on": []any{"config", "mesh-resolver.nodes"}}, + }, + }}}.Declaration(Rendering{}) + if err != nil { + t.Fatal(err) + } + for _, r := range out { + if r["type"] != "service" { + continue + } + reflects := r["restart-on"].([]any) + var own, other bool + for _, id := range reflects { + switch id.(string) { + case "dnsmasq.config": + own = true + case "mesh-resolver.nodes": + other = true + } + } + if !own { + t.Fatalf("its own file was not prefixed with its module: %v", reflects) + } + if !other { + t.Fatalf("a file it named in full was prefixed again and now points at nothing: %v", + reflects) + } + return + } + t.Fatal("no service in the declaration") +}