diff --git a/examples/modules/README.md b/examples/modules/README.md new file mode 100644 index 0000000..5962d21 --- /dev/null +++ b/examples/modules/README.md @@ -0,0 +1,42 @@ +# Example modules + +Manifests, not programs. They are here because the contract is easier to read as something that +works than as a description of something that would. + +**Third-party software runs *on* the mesh, not *of* it** (novox/hq ADR 0001). dnsmasq is not the +mesh's, and neither is systemd-resolved — what is the mesh's is the fact only it can know, which +is which machines exist and where they are. So the mesh writes that to a file and these read it. + +## Resolving a service named under a machine + +`postgres.novox.internal`, `plex.ace.internal`. The first label is the service and the rest is the +node, so **anything under a node's name must resolve to that node** and a proxy there routes by +the name it was asked for. That routing is a separate concern and stays separate. + +Two roles, and they are genuinely different things: + +| | claims | | +|---|---|---| +| **serving** | `the-dns-port` | answers the wildcards — `dnsmasq.json` | +| **asking** | `the-resolver-configuration` | decides what the machine asks — `resolved-split-dns.json`, `resolv-conf.json` | + +**systemd-resolved cannot serve a wildcard**, so it is only ever an *asking* module: it routes the +mesh's suffix to something that can. Treating the two roles as one would produce a module that +cannot work, which is the mistake worth naming. + +Assign one of each. Two of either is refused by the mesh rather than fought over on the machine: + +> `resolved-split-dns and resolv-conf both claim "the-resolver-configuration", and only one thing +> may hold it per node` + +ADR 0009's table names that resource `/etc/resolv.conf`, which is what it *is*, the way it writes +*the seat*. A claim is a name in the catalogue's own form, so it is written as one. + +## Why neither needs to know the machine's address + +Both would ordinarily need it — a resolver must bind somewhere, and a stub must be pointed +somewhere — and a static manifest cannot know it. + +Neither does, because both name things **the mesh itself named**: the private network's interface +is `mesh0` on every machine, and the address a resolver listens on for the machine's own use is +`127.0.0.54` on every machine. A name the mesh chose is a name a manifest can use. diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json new file mode 100644 index 0000000..15d2059 --- /dev/null +++ b/examples/modules/dnsmasq.json @@ -0,0 +1,24 @@ +{ + "module": "dnsmasq", + "version": "1", + + "requires": ["resolver-data"], + "provides": ["wildcard-resolution"], + "claims": [{"name": "the-dns-port", "scope": "node"}], + + "listens": [ + {"port": 53, "protocol": "udp", "from": "mesh", + "why": "names under every machine in this mesh, for this machine and what it runs"} + ], + + "resources": [ + {"id": "package", "type": "package", "package": "dnsmasq"}, + + {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it — including a container\n# on this machine — can ask.\n# 127.0.0.54 this machine's own use. Not 127.0.0.1 and not 127.0.0.53:\n# the first is where everything else expects a resolver, and the\n# second is systemd-resolved's. Taking either would be this\n# module claiming something it did not say it claims.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.54\n\n# It answers for the mesh and forwards nothing it was not asked about. Names\n# outside the mesh are somebody else's business, and a resolver that answered\n# them would be this module taking over more than it claims.\ndomain-needed\nbogus-priv\n"}, + + {"id": "service", "type": "service", "unit": "dnsmasq.service", + "state": "running", "boot": "enabled", + "restart-on": ["config", "mesh-resolver.nodes"]} + ] +} diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go new file mode 100644 index 0000000..530e862 --- /dev/null +++ b/examples/modules/modules_test.go @@ -0,0 +1,157 @@ +package modules + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/overlay" +) + +// The examples are manifests, so the thing to check is that the catalogue accepts them. +// +// A manifest that only ever appears in a document is a manifest nobody has run through the parser, +// and the parser refuses unknown keys — so a typo here would be discovered by whoever first tried +// to use one, which is the opposite of what an example is for. +func read(t *testing.T, name string) catalogue.Manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join(".", name)) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("%s is not a manifest this mesh accepts: %v", name, err) + } + return m +} + +func TestEveryExampleIsAManifestTheMeshAccepts(t *testing.T) { + found, err := filepath.Glob("*.json") + if err != nil { + t.Fatal(err) + } + if len(found) == 0 { + t.Fatal("no examples, so this test proves nothing") + } + for _, name := range found { + read(t, name) + } +} + +// The serving module reads what the mesh writes, and restarts when the mesh rewrites it. +// +// Without the second it would serve the names it started with for ever — every machine that +// joined afterwards unreachable by name, and every check passing. +func TestTheResolverReadsTheMeshsNamesAndFollowsThem(t *testing.T) { + m := read(t, "dnsmasq.json") + + var config, service map[string]any + for _, r := range m.Resources { + switch r["id"] { + case "config": + config = r + case "service": + service = r + } + } + if config == nil || service == nil { + t.Fatal("the module has no configuration or no service") + } + if !strings.Contains(config["content"].(string), overlay.ResolverPath) { + t.Fatalf("it does not read what the mesh writes at %s", overlay.ResolverPath) + } + + var follows bool + for _, id := range service["restart-on"].([]any) { + if id.(string) == overlay.Resolver+".nodes" { + follows = true + } + } + if !follows { + t.Fatalf("it does not restart when the mesh rewrites the names: %v", service["restart-on"]) + } +} + +// It binds names the mesh chose, so it needs to know nothing about the machine it is on. +// +// That is the whole reason these can be static manifests: a resolver must bind somewhere and a +// stub must be pointed somewhere, and neither address is knowable in advance — unless the mesh +// named it. +func TestTheResolverNeedsToKnowNothingAboutItsMachine(t *testing.T) { + config := read(t, "dnsmasq.json").Resources[1]["content"].(string) + // The directive, not the word: the comment above it names the interface too, so a plain + // Contains passes whatever the module actually binds. It did. + if !strings.Contains(config, "interface="+overlay.Interface+"\n") { + t.Fatalf("it does not bind the private network's interface %q:\n%s", + overlay.Interface, config) + } + if !strings.Contains(config, "listen-address=127.0.0.54\n") { + t.Fatalf("it does not answer on the address the asking modules point at:\n%s", config) + } + // Not the two addresses that belong to something else. + for _, taken := range []string{"listen-address=127.0.0.1", "listen-address=127.0.0.53"} { + if strings.Contains(config, taken) { + t.Fatalf("it takes %q, which belongs to something it did not claim:\n%s", taken, config) + } + } +} + +// The two ways of deciding what a machine asks claim the same thing, so the mesh refuses the pair. +func TestTwoWaysOfOwningTheResolverCannotBothBeAssigned(t *testing.T) { + shelf := map[string]catalogue.Manifest{} + for _, name := range []string{"resolved-split-dns.json", "resolv-conf.json", "dnsmasq.json"} { + m := read(t, name) + shelf[m.Module] = m + } + // Something has to answer `wildcard-resolution`, or they are refused for that instead and the + // test would pass without ever reaching the claim. + shelf["dnsmasq"] = read(t, "dnsmasq.json") + + _, err := catalogue.Resolve(shelf, + []string{"dnsmasq", "resolved-split-dns", "resolv-conf"}, + catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, + catalogue.World{Unchecked: true}) + if err == nil { + t.Fatal("both ways of owning the resolver were assigned to one machine") + } + said := err.Error() + if !strings.Contains(said, "the-resolver-configuration") { + t.Fatalf("the refusal does not name what they both want: %v", said) + } +} + +// And the two roles are not the same claim: a machine runs one resolver AND one thing deciding +// what it asks, so serving and asking must be assignable together. +func TestServingAndAskingAreAssignableTogether(t *testing.T) { + shelf := map[string]catalogue.Manifest{} + for _, name := range []string{"dnsmasq.json", "resolved-split-dns.json"} { + m := read(t, name) + shelf[m.Module] = m + } + if _, err := catalogue.Resolve(shelf, + []string{"dnsmasq", "resolved-split-dns"}, + catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, + catalogue.World{Unchecked: true}); err != nil { + t.Fatalf("a resolver and the thing pointing at it cannot both be assigned: %v", err) + } +} + +// The examples are JSON a person edits, so a stray comma is worth catching here rather than on a +// machine. +func TestTheExamplesAreWellFormed(t *testing.T) { + found, _ := filepath.Glob("*.json") + for _, name := range found { + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + var any map[string]any + if err := json.Unmarshal(raw, &any); err != nil { + t.Fatalf("%s is not JSON: %v", name, err) + } + } +} diff --git a/examples/modules/resolv-conf.json b/examples/modules/resolv-conf.json new file mode 100644 index 0000000..5631573 --- /dev/null +++ b/examples/modules/resolv-conf.json @@ -0,0 +1,12 @@ +{ + "module": "resolv-conf", + "version": "1", + + "requires": ["wildcard-resolution"], + "claims": [{"name": "the-resolver-configuration", "scope": "node"}], + + "resources": [ + {"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644", + "content": "# Managed by the mesh.\n#\n# For a machine where nothing else owns this file. On one where systemd-resolved\n# or NetworkManager does, assign that module instead — this one and those claim\n# the same thing, so the mesh refuses the pair rather than letting them take\n# turns overwriting each other, which is the failure this claim exists to stop.\n#\n# The mesh's resolver first, because it answers only the mesh's names and\n# forwards nothing: a query it does not recognise falls through to the next\n# line rather than being answered wrongly.\nnameserver 127.0.0.54\n\n# And what this machine used before. Replace this line with the resolver this\n# machine should use for everything that is not the mesh — it is not the mesh's\n# to choose, and a public one written here by default would send every query\n# this machine makes somewhere nobody agreed to.\nnameserver 127.0.0.53\n"} + ] +} diff --git a/examples/modules/resolved-split-dns.json b/examples/modules/resolved-split-dns.json new file mode 100644 index 0000000..cbe99ea --- /dev/null +++ b/examples/modules/resolved-split-dns.json @@ -0,0 +1,18 @@ +{ + "module": "resolved-split-dns", + "version": "1", + + "requires": ["wildcard-resolution"], + "claims": [{"name": "the-resolver-configuration", "scope": "node"}], + + "resources": [ + {"id": "drop-in", "type": "directory", "path": "/etc/systemd/resolved.conf.d", "mode": "0755"}, + + {"id": "route", "type": "file", + "path": "/etc/systemd/resolved.conf.d/mesh.conf", "mode": "0644", + "content": "# Managed by the mesh.\n#\n# **Only the mesh's names.** The tilde makes this a routing domain rather than a\n# search domain: queries under it go to the resolver below, and everything else\n# keeps going wherever this machine already sent it. A resolver that took over\n# all of DNS would be this module claiming the machine's whole network, which\n# is not what it says it claims.\n#\n# 127.0.0.54 is where the mesh's resolver answers on every machine — a name the\n# mesh chose, so this file needs to know nothing about this particular one.\n[Resolve]\nDNS=127.0.0.54\nDomains=~internal\n"}, + + {"id": "resolved", "type": "service", "unit": "systemd-resolved.service", + "state": "running", "boot": "enabled", "restart-on": ["route"]} + ] +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 6cefc81..1b62383 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -314,10 +314,22 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those // are prefixed too or they would point at nothing. + // + // **Unless it already names one.** A module may reflect a file another module put on + // the machine — the case this exists for is a resolver restarting when the mesh + // rewrites the names, which are computed by the mesh and belong to its module, not to + // the daemon's. Written `.`, and a dot is what marks it as already + // answered: prefixing it again would point at nothing, silently, and the daemon would + // serve the old names for ever while everything reported success. if reflects, ok := resource["restart-on"].([]any); ok { var renamed []any for _, id := range reflects { - renamed = append(renamed, m.Module+"."+fmt.Sprint(id)) + named := fmt.Sprint(id) + if strings.Contains(named, ".") { + renamed = append(renamed, named) + continue + } + renamed = append(renamed, m.Module+"."+named) } copied["restart-on"] = renamed } diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index b029ca2..408f376 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -458,3 +458,47 @@ func TestAGeneratorThatOpensNothingNeedsNoMethod(t *testing.T) { t.Fatalf("a generator that says nothing about ports opened one: %+v", rules) } } + +// A module may reflect a file another module put on the machine. +// +// The case this exists for: a resolver restarting when the mesh rewrites the names. Those are +// computed by the mesh and belong to its module, not to the daemon's — so without this the +// daemon would serve the names it started with for ever, while every check reported success. +func TestAServiceCanReflectAnotherModulesFile(t *testing.T) { + out, err := Resolution{Node: "anchor", Modules: []Manifest{{ + Module: "dnsmasq", + Resources: []map[string]any{ + {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", + "content": "x", "mode": "0644"}, + {"id": "run", "type": "service", "unit": "dnsmasq.service", "state": "running", + "restart-on": []any{"config", "mesh-resolver.nodes"}}, + }, + }}}.Declaration(Rendering{}) + if err != nil { + t.Fatal(err) + } + for _, r := range out { + if r["type"] != "service" { + continue + } + reflects := r["restart-on"].([]any) + var own, other bool + for _, id := range reflects { + switch id.(string) { + case "dnsmasq.config": + own = true + case "mesh-resolver.nodes": + other = true + } + } + if !own { + t.Fatalf("its own file was not prefixed with its module: %v", reflects) + } + if !other { + t.Fatalf("a file it named in full was prefixed again and now points at nothing: %v", + reflects) + } + return + } + t.Fatal("no service in the declaration") +}