From 1f5b70a9953a8af31ea8e5f7780a02b76177c429 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 1 Sep 2026 17:52:53 +0200 Subject: [PATCH] The mesh assigns the port, and a module says it once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit novox/hq ADR 0038. A module cannot choose a port: it is written once and assigned anywhere, so any number it picks is a guess about a machine it has never seen. A database module met the mesh's own store on 5432 and was told, by a container runtime three layers down, that the port was already allocated. The number used to appear three times in every module — the rule set, what a consumer is told, and what the runtime publishes — agreeing only because one person wrote all three. Now it appears once, in `listens`, and the other two are derived: the container publishes `20000:5432`, the consumer is told 20000, and the rule set opens 20000. An assignment is made once and kept, as a credential is. A port that moved on every declaration would restart both ends each time and hand a consumer a number that was true when it was read. Ports the protocol fixes — mail on 25, submission on 587, DNS on 53 — say so, and are then claims: one holder per machine, and the second is refused by name at assignment. That is the mechanism the mesh already has for what is singular on a machine, pointed at ports. A mapping written the long way is left exactly as it is. Some things must be pinned by hand, and quietly overruling somebody who wrote both halves would be worse than not offering the short form. Still open, and known: the substrate is not a module, so the mesh has never heard of its own store and cannot yet assign around it. That is what 028 will still be about after this. --- cmd/mesh-control/plan.go | 47 ++++- examples/modules/dnsmasq.json | 58 ++++-- examples/modules/gitea.json | 2 +- examples/modules/keycloak.json | 2 +- examples/modules/mailu.json | 20 +- examples/modules/minio.json | 3 +- examples/modules/modules_test.go | 7 +- examples/modules/object-store.json | 3 +- examples/modules/postgres.json | 6 +- internal/catalogue/declaration.go | 112 ++++++++++- internal/catalogue/filtering.go | 14 +- internal/catalogue/filtering_test.go | 4 +- internal/catalogue/manifest.go | 10 + internal/catalogue/resolve.go | 6 +- .../0016-the-mesh-assigns-the-port.sql | 33 ++++ internal/inventory/ports.go | 186 ++++++++++++++++++ internal/inventory/ports_test.go | 151 ++++++++++++++ 17 files changed, 617 insertions(+), 47 deletions(-) create mode 100644 internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql create mode 100644 internal/inventory/ports.go create mode 100644 internal/inventory/ports_test.go diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index e9f1579..be10c90 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -196,7 +196,11 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, // What that module says a consumer needs to know, with that node's settings on // it: a port somebody moved on the provider is a port its consumers must be told // about, and the two coming from different places is how they come to disagree. - serves := m.Serves[name] + assigned, err := portsOn(ctx, inv, o.node.Name, m.Module) + if err != nil { + return catalogue.World{}, err + } + serves := catalogue.ServedOn(m, name, assigned) if len(serves) > 0 { layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) if err != nil { @@ -253,6 +257,28 @@ func declarationWith(ctx context.Context, open *stores, node string, if err != nil { return nil, err } + // Where this machine puts what each module needs reachable (novox/hq ADR 0038). + // + // **Assigned here rather than written by a module**, because a module is written once and + // assigned anywhere: any number it picks is a guess about a machine it has never seen. Made + // before the declaration is composed, because the container's mapping, the rule set and what a + // consumer is told are all derived from it. + ports := map[string]map[int]int{} + for _, m := range plan.Modules { + for _, l := range m.Listens { + at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed) + if err != nil { + return nil, fmt.Errorf( + "%s needs %d reachable on %s and it could not be assigned: %w", + m.Module, l.Port, node, err) + } + if ports[m.Module] == nil { + ports[m.Module] = map[int]int{} + } + ports[m.Module][l.Port] = at.Machine + } + } + // And each module's own secrets — a superuser password, an administrator, an account. Made // per node, so a module running on three machines has three. needed := map[string]map[string]string{} @@ -302,7 +328,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } return plan.Declaration(catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names}) } @@ -562,3 +588,20 @@ func keyFor(ctx context.Context, open *stores, licence, node, module string) (st } return held.KeyFor(ctx, licence, node, module) } + +// portsOn is one module's assignments on one machine, by the port the software uses. +func portsOn( + ctx context.Context, inv *inventory.Inventory, node, module string, +) (map[int]int, error) { + all, err := inv.PortsFor(ctx, node) + if err != nil { + return nil, err + } + out := map[int]int{} + for _, a := range all { + if a.Module == module { + out[a.Wanted] = a.Machine + } + } + return out, nil +} diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json index 7ed1ff6..79d311d 100644 --- a/examples/modules/dnsmasq.json +++ b/examples/modules/dnsmasq.json @@ -1,24 +1,50 @@ { "module": "dnsmasq", "version": "1", - - "requires": ["resolver-data"], - "provides": ["wildcard-resolution"], - "claims": [{"name": "the-dns-port", "scope": "node"}], - + "requires": [ + "resolver-data" + ], + "provides": [ + "wildcard-resolution" + ], + "claims": [ + { + "name": "the-dns-port", + "scope": "node" + } + ], "listens": [ - {"port": 53, "protocol": "udp", "from": "mesh", - "why": "names under every machine in this mesh, for this machine and what it runs"} + { + "port": 53, + "protocol": "udp", + "from": "mesh", + "why": "names under every machine in this mesh, for this machine and what it runs", + "fixed": true + } ], - "resources": [ - {"id": "package", "type": "package", "package": "dnsmasq"}, - - {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it — including a container\n# on this machine — can ask.\n# 127.0.0.55 this machine's own use, for whatever points resolution at the\n# mesh. Not .53 or .54: systemd-resolved holds BOTH — .53 is its\n# stub and .54 its proxy stub — which this module asserted was\n# free until a machine said otherwise.\n#\n# Listening on a loopback address makes dnsmasq take the rest of\n# loopback with it, 127.0.0.1 included. That is why this module\n# claims `the-dns-port`: it takes the machine's DNS port, and\n# saying it takes only one address would be the same kind of\n# comfortable claim that .54 was free.\n#\n# .55 is a convention and not a reservation. If a future systemd\n# takes it, this line changes and nothing else does, which is the\n# reason it is written once here rather than in each module that\n# points at it.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.55\n\n# **It forwards nothing, and must not read resolv.conf to find out where to.**\n# Whatever points this machine at the mesh writes its own address into\n# resolv.conf — so a resolver that read it for upstreams would find itself,\n# and every query it could not answer locally would loop until its receive\n# queue filled. That is not theoretical: it filled with 15KB of queries and\n# every lookup on the machine hung.\n#\n# It needs no upstream because it is never asked for anything else: the\n# asking module routes only the mesh's suffix here and leaves the rest\n# wherever the machine already sent it.\nno-resolv\ndomain-needed\nbogus-priv\n"}, - - {"id": "service", "type": "service", "unit": "dnsmasq.service", - "state": "running", "boot": "enabled", - "restart-on": ["config", "mesh-resolver.nodes"]} + { + "id": "package", + "type": "package", + "package": "dnsmasq" + }, + { + "id": "config", + "type": "file", + "path": "/etc/dnsmasq.conf", + "mode": "0644", + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it \u2014 including a container\n# on this machine \u2014 can ask.\n# 127.0.0.55 this machine's own use, for whatever points resolution at the\n# mesh. Not .53 or .54: systemd-resolved holds BOTH \u2014 .53 is its\n# stub and .54 its proxy stub \u2014 which this module asserted was\n# free until a machine said otherwise.\n#\n# Listening on a loopback address makes dnsmasq take the rest of\n# loopback with it, 127.0.0.1 included. That is why this module\n# claims `the-dns-port`: it takes the machine's DNS port, and\n# saying it takes only one address would be the same kind of\n# comfortable claim that .54 was free.\n#\n# .55 is a convention and not a reservation. If a future systemd\n# takes it, this line changes and nothing else does, which is the\n# reason it is written once here rather than in each module that\n# points at it.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.55\n\n# **It forwards nothing, and must not read resolv.conf to find out where to.**\n# Whatever points this machine at the mesh writes its own address into\n# resolv.conf \u2014 so a resolver that read it for upstreams would find itself,\n# and every query it could not answer locally would loop until its receive\n# queue filled. That is not theoretical: it filled with 15KB of queries and\n# every lookup on the machine hung.\n#\n# It needs no upstream because it is never asked for anything else: the\n# asking module routes only the mesh's suffix here and leaves the rest\n# wherever the machine already sent it.\nno-resolv\ndomain-needed\nbogus-priv\n" + }, + { + "id": "service", + "type": "service", + "unit": "dnsmasq.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "config", + "mesh-resolver.nodes" + ] + } ] } diff --git a/examples/modules/gitea.json b/examples/modules/gitea.json index 987add6..c307925 100644 --- a/examples/modules/gitea.json +++ b/examples/modules/gitea.json @@ -69,7 +69,7 @@ "/var/lib/gitea/server.env" ], "ports": [ - "3000:3000", + "3000", "2222:22" ], "volumes": [ diff --git a/examples/modules/keycloak.json b/examples/modules/keycloak.json index 530dbf0..5096b20 100644 --- a/examples/modules/keycloak.json +++ b/examples/modules/keycloak.json @@ -74,7 +74,7 @@ "/var/lib/keycloak/database.env" ], "ports": [ - "8080:8080" + "8080" ] } ] diff --git a/examples/modules/mailu.json b/examples/modules/mailu.json index c0d7a2c..b1fe323 100644 --- a/examples/modules/mailu.json +++ b/examples/modules/mailu.json @@ -9,25 +9,29 @@ "port": 25, "protocol": "tcp", "from": "anywhere", - "why": "mail from other mail servers" + "why": "mail from other mail servers", + "fixed": true }, { "port": 465, "protocol": "tcp", "from": "anywhere", - "why": "submission over TLS" + "why": "submission over TLS", + "fixed": true }, { "port": 587, "protocol": "tcp", "from": "anywhere", - "why": "submission" + "why": "submission", + "fixed": true }, { "port": 993, "protocol": "tcp", "from": "anywhere", - "why": "IMAP over TLS" + "why": "IMAP over TLS", + "fixed": true }, { "port": 7080, @@ -255,10 +259,10 @@ "/var/lib/mailu/secret.env" ], "ports": [ - "25:25", - "465:465", - "587:587", - "993:993", + "25", + "465", + "587", + "993", "7080:80" ], "volumes": [ diff --git a/examples/modules/minio.json b/examples/modules/minio.json index 1feb6a5..12c285a 100644 --- a/examples/modules/minio.json +++ b/examples/modules/minio.json @@ -20,7 +20,6 @@ ], "serves": { "s3-bucket": { - "port": 9000, "scheme": "http", "region": "us-east-1" } @@ -81,7 +80,7 @@ "/var/lib/minio/root.env" ], "ports": [ - "9000:9000" + "9000" ], "volumes": [ "/services/minio/data/data1-1:/data" diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index cbd1a2a..c152f61 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -429,10 +429,13 @@ func declareOnItsOwn(t *testing.T, shelf map[string]catalogue.Manifest, // providing example says it serves. offered := map[string][]catalogue.Provider{} for _, want := range m.Requires { + // Built the way the control plane builds it: what a provider tells a consumer includes + // the port, and the module no longer writes that into `serves` by hand — it says it once + // in `listens` and the mesh puts it there (novox/hq ADR 0038). serves := map[string]any{} for _, other := range shelf { - if s, said := other.Serves[want]; said { - serves = s + if _, said := other.Serves[want]; said { + serves = catalogue.ServedOn(other, want, nil) } } offered[want] = []catalogue.Provider{ diff --git a/examples/modules/object-store.json b/examples/modules/object-store.json index 22243b8..5d2d3df 100644 --- a/examples/modules/object-store.json +++ b/examples/modules/object-store.json @@ -20,7 +20,6 @@ ], "serves": { "s3-bucket": { - "port": 9000, "scheme": "http", "region": "us-east-1" } @@ -60,7 +59,7 @@ "MINIO_ROOT_USER": "meshroot" }, "ports": [ - "9000:9000" + "9000" ], "volumes": [ "mesh-store-data:/data", diff --git a/examples/modules/postgres.json b/examples/modules/postgres.json index 7bd4edb..e2d8f03 100644 --- a/examples/modules/postgres.json +++ b/examples/modules/postgres.json @@ -19,9 +19,7 @@ } ], "serves": { - "postgres-database": { - "port": 5432 - } + "postgres-database": {} }, "receives": { "postgres-database": "/var/lib/postgres/grants/mesh.json" @@ -77,7 +75,7 @@ "/var/lib/postgres/superuser.env" ], "ports": [ - "5432:5432" + "5432" ], "volumes": [ "/services/postgres/db-data:/var/lib/postgresql/data" diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 7395288..ea5d8ed 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "strings" ) @@ -97,6 +98,24 @@ type Rendering struct { // rather than resolved, because who consumes a node is a fact about the rest of the mesh and // resolution answers questions about one machine. Grants []Grant + + // Ports is where this machine puts what each module needs reachable, by module and by the + // port the software itself uses (novox/hq ADR 0038). + // + // **The one place the number now lives.** A module used to write it three times — for the rule + // set, for what a consumer is told, and for what the runtime publishes — and nothing checked + // that the three agreed. They are all derived from this. + Ports map[string]map[int]int +} + +// machinePort is where a module's port lives on this machine, or the port itself when the mesh has +// not been asked. Unassigned is not an error here: a module with no `listens` never needed one, +// and a caller composing a declaration without a store still gets something coherent. +func (r Rendering) machinePort(module string, wanted int) int { + if at, known := r.Ports[module][wanted]; known { + return at + } + return wanted } // Declaration is everything the resolved modules put on the node, with settings applied. @@ -120,7 +139,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // Once, from every module's listens -- not per module. A module receiving only its own ports // would write a rule set that closed every other module on the machine. - rules, err := r.Filtering(with.Generators) + rules, err := r.Filtering(with.Generators, with.Ports) if err != nil { return nil, err } @@ -345,6 +364,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if err := pinned(copied, m.Module); err != nil { return nil, err } + publishedOn(copied, m.Module, with) copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those // are prefixed too or they would point at nothing. @@ -593,7 +613,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti // nothing resolves would be worse than naming it by an address that always works. func here(r Resolution, requirement string) *Needed { for _, m := range r.Modules { - serves, said := m.Serves[requirement] + _, said := m.Serves[requirement] + serves := ServedOn(m, requirement, nil) if !said || len(serves) == 0 { continue } @@ -667,3 +688,90 @@ func pinned(resource map[string]any, module string) error { "image — it would be fetched and fail there. Resolve the tag to a digest first", module, resource["id"]) } + +// publishedOn puts the machine's own port on the outside of a container's mapping. +// +// **A module writes the port the software uses; the mesh says where the machine puts it** +// (novox/hq ADR 0038). So `"5432"` means *publish what the software calls 5432*, and this fills in +// the half only the mesh can know. +// +// A mapping written the long way — `"5433:5432"` — is left exactly as it is. Some things genuinely +// must be pinned down by hand, and quietly overruling somebody who wrote both halves would be +// worse than not offering the short form at all. +func publishedOn(resource map[string]any, module string, with Rendering) { + if fmt.Sprint(resource["type"]) != "container" { + return + } + listed, ok := resource["ports"].([]any) + if !ok { + return + } + out := make([]any, 0, len(listed)) + for _, entry := range listed { + written := fmt.Sprint(entry) + if strings.Contains(written, ":") { + out = append(out, written) + continue + } + wanted, err := strconv.Atoi(strings.TrimSpace(written)) + if err != nil { + // Not a port at all. Passed through, so the host refuses it with its own words rather + // than this quietly dropping something somebody meant. + out = append(out, written) + continue + } + out = append(out, fmt.Sprintf("%d:%d", with.machinePort(module, wanted), wanted)) + } + resource["ports"] = out +} + +// ServedOn is what a provider tells a consumer, with the port that machine actually uses. +// +// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three +// times — for the rule set, for what a consumer is told, and for what the runtime publishes — and +// nothing checked that the three agreed. This is what fills the second in. +// +// The assignment wins when there is one, and the declared port stands in when there is not: a +// caller composing without a store still gets something coherent, and a mesh that has assigned one +// tells the truth about where it put it. +// +// Only when the module offers exactly one port. A module offering several has not said which +// belongs to which provision, and guessing would give a consumer a port that answers something +// else — so it keeps whatever the manifest said, which may be nothing. +func ServedOn(m Manifest, provision string, ports map[int]int) map[string]any { + serves := m.Serves[provision] + + out := make(map[string]any, len(serves)+1) + for k, v := range serves { + out[k] = v + } + if _, said := out["port"]; said { + // Written by hand. Redirected to wherever the machine put it, and otherwise left alone. + if number, ok := asPort(out["port"]); ok { + if at, known := ports[number]; known { + out["port"] = at + } + } + return out + } + if len(m.Listens) != 1 { + return out + } + wanted := m.Listens[0].Port + if at, known := ports[wanted]; known { + out["port"] = at + } else { + out["port"] = wanted + } + return out +} + +func asPort(v any) (int, bool) { + switch n := v.(type) { + case int: + return n, true + case float64: + return int(n), true + } + return 0, false +} diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index b224665..36a1aba 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -31,7 +31,7 @@ type Rule struct { // a consequence of what runs on it, not a second list kept in step by hand. Nothing else opens a // port: **what is not declared is closed**, which is the property that makes the derivation worth // having rather than merely tidy. -func (r Resolution) Filtering(computed map[string]Generator) ([]Rule, error) { +func (r Resolution) Filtering(computed map[string]Generator, ports map[string]map[int]int) ([]Rule, error) { // Keyed by what actually distinguishes an opening. Two modules wanting :443 from the mesh is // one rule with two sources; one wanting it from the mesh and another from anywhere is two, // and they are collapsed below -- deliberately, and only in the widening direction. @@ -62,10 +62,18 @@ func (r Resolution) Filtering(computed map[string]Generator) ([]Rule, error) { } } for _, l := range opens { - at := opening{port: l.Port, protocol: l.At(), from: l.From} + // **The port the machine actually publishes on** (novox/hq ADR 0038). A module says + // the port its software uses; the mesh chooses where the machine puts it. A rule + // naming the first would open a port nothing listens on and leave the real one shut, + // which is a firewall that reports success and blocks the service. + port := l.Port + if at, known := ports[m.Module][l.Port]; known { + port = at + } + at := opening{port: port, protocol: l.At(), from: l.From} rule, seen := found[at] if !seen { - rule = &Rule{Port: l.Port, Protocol: l.At(), From: l.From} + rule = &Rule{Port: port, Protocol: l.At(), From: l.From} found[at] = rule } rule.Because = append(rule.Because, m.Module) diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index 1d9669d..204e37b 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -369,7 +369,7 @@ func TestAMachineOffTheNetworkIsBoundToItselfByAnAddressThatWorks(t *testing.T) // mustFilter is the rule set, refusing to continue if it could not be computed. func mustFilter(t *testing.T, r Resolution, computed map[string]Generator) []Rule { t.Helper() - rules, err := r.Filtering(computed) + rules, err := r.Filtering(computed, nil) if err != nil { t.Fatalf("no rule set could be computed: %v", err) } @@ -440,7 +440,7 @@ func TestAComputedModulesOwnListensAreNotLost(t *testing.T) { func TestAGeneratorThatCannotSayRefusesTheRuleSet(t *testing.T) { _, err := Resolution{Node: "anchor", Modules: []Manifest{{Module: "networking", Computed: "mesh-network"}}, - }.Filtering(map[string]Generator{"mesh-network": cannotSay{}}) + }.Filtering(map[string]Generator{"mesh-network": cannotSay{}}, nil) if err == nil { t.Fatal("a machine whose open ports could not be computed was given a rule set anyway") } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index a1f4505..c151e00 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -318,6 +318,16 @@ type Listening struct { // From is who may reach it. Required, because a rule with no source is open and must say so // rather than appear to restrict something. From string `json:"from"` + // Fixed means the protocol chose this number, so the machine must use it too. + // + // **The exception, and it is a real one** (novox/hq ADR 0038). Mail is 25, submission is 587, + // IMAP over TLS is 993 — a mail system on a port the mesh picked is a mail system nothing can + // deliver to. Everything else the mesh assigns, because a module cannot know what else is on + // the machine it lands on. + // + // A fixed port is a **claim**: one holder per machine, and the second is refused by name when + // it is assigned rather than by a container runtime when it is applied. + Fixed bool `json:"fixed,omitempty"` // Why this port is open, for somebody reading a generated rule set and wondering. Why string `json:"why,omitempty"` } diff --git a/internal/catalogue/resolve.go b/internal/catalogue/resolve.go index a28eda2..13288b4 100644 --- a/internal/catalogue/resolve.go +++ b/internal/catalogue/resolve.go @@ -474,8 +474,10 @@ func servedHere(catalogue map[string]Manifest, chosen map[string]bool, want stri if !chosen[name] { continue } - if serves, ok := m.Serves[want]; ok { - return serves + if _, ok := m.Serves[want]; ok { + // Without assignments: this is resolution, which runs before a machine's ports are + // known. The declaration fills the machine's own in afterwards, where it has them. + return ServedOn(m, want, nil) } } return nil diff --git a/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql b/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql new file mode 100644 index 0000000..ac966d5 --- /dev/null +++ b/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql @@ -0,0 +1,33 @@ +-- Which port a machine uses for what a module needs reachable. +-- +-- novox/hq ADR 0038. A module cannot choose this: it is written once and assigned anywhere, so any +-- number it picks is a guess about a machine it has never seen. Two modules guessing the same one +-- is not a mistake either of them made -- it is a database module meeting the mesh's own store and +-- being told, by a container runtime three layers down, that the port is already allocated. +-- +-- **Made once and kept**, exactly as a credential is. A port that moved on every declaration would +-- restart both ends each time, and would hand a consumer a number that was true when it was read. + +create table port_assignment ( + node uuid not null references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + + -- The port the software itself uses -- what a module writes down, and the only part it knows. + wanted integer not null, + -- What the machine publishes it on. The same as `wanted` when the protocol fixes it. + machine integer not null, + + -- Whether the protocol fixed it. Kept rather than derived: *this is 25 because it must be* + -- and *this is 25 because it was free* are different facts, and only the first refuses a + -- second holder. + fixed boolean not null default false, + + assigned_at timestamptz not null default now(), + + primary key (node, module, wanted), + + -- **One machine port, one holder.** The constraint is the point: a second module cannot be + -- given a port the first has, and finding that out here is finding it out at assignment + -- rather than at apply. + unique (node, machine) +); diff --git a/internal/inventory/ports.go b/internal/inventory/ports.go new file mode 100644 index 0000000..52d8b23 --- /dev/null +++ b/internal/inventory/ports.go @@ -0,0 +1,186 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// Which port a machine uses for what a module needs reachable. +// +// **A module cannot choose this** (novox/hq ADR 0038). It is written once and assigned anywhere, +// so any number it picks is a guess about a machine it has never seen. The mesh is the only thing +// that knows what else is there, so the mesh chooses — and a module says only that something must +// be reachable, and what the software itself calls it. + +// Assigned is where the machine puts one of a module's ports. +type Assigned struct { + Module string + // Wanted is the port the software uses — what the module wrote down. + Wanted int + // Machine is where this machine publishes it. + Machine int + // Fixed means the protocol chose it, not the mesh. + Fixed bool +} + +// The range the mesh assigns from. +// +// High and unprivileged, so an assignment never needs root and never lands on something a person +// would recognise. Below the range Linux uses for outgoing connections, so an assignment cannot +// collide with a port the kernel handed to something else while the machine was working. +const ( + firstAssignable = 20000 + lastAssignable = 29999 +) + +// ErrPortTaken is returned when a port the protocol fixes is already held by another module. +var ErrPortTaken = errors.New("that port is already held on this machine") + +// PortFor is where one module's port lives on one machine, choosing it the first time. +// +// **Kept once chosen.** A port that moved on every declaration would restart both ends each time, +// and would hand a consumer a number that was true when it was read — which is the same argument +// that makes a credential stable. +func (i *Inventory) PortFor( + ctx context.Context, node, module string, wanted int, fixed bool, +) (Assigned, error) { + record, err := i.NodeByName(ctx, node) + if err != nil { + return Assigned{}, err + } + + var held Assigned + err = i.store.Pool().QueryRow(ctx, + `select machine, fixed from port_assignment + where node = $1 and module = $2 and wanted = $3`, + record.ID, module, wanted).Scan(&held.Machine, &held.Fixed) + if err == nil { + held.Module, held.Wanted = module, wanted + // A module that has become fixed since it was assigned must move to the port the protocol + // requires. Said rather than silently kept: a mail system on the port it was lent is a + // mail system nothing can deliver to. + if fixed && held.Machine != wanted { + if err := i.freePort(ctx, record.ID, module, wanted); err != nil { + return Assigned{}, err + } + return i.assignPort(ctx, record.ID, node, module, wanted, true) + } + return held, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return Assigned{}, err + } + return i.assignPort(ctx, record.ID, node, module, wanted, fixed) +} + +func (i *Inventory) freePort(ctx context.Context, node any, module string, wanted int) error { + _, err := i.store.Pool().Exec(ctx, + `delete from port_assignment where node = $1 and module = $2 and wanted = $3`, + node, module, wanted) + return err +} + +func (i *Inventory) assignPort( + ctx context.Context, nodeID any, node, module string, wanted int, fixed bool, +) (Assigned, error) { + taken, err := i.portsOn(ctx, nodeID) + if err != nil { + return Assigned{}, err + } + + machine := wanted + if !fixed { + // The lowest free one, so a machine's assignments are stable and readable rather than + // scattered — and so the same set of modules on two machines gets the same numbers, which + // makes a difference between two machines mean something. + machine = 0 + for candidate := firstAssignable; candidate <= lastAssignable; candidate++ { + if _, held := taken[candidate]; !held { + machine = candidate + break + } + } + if machine == 0 { + return Assigned{}, fmt.Errorf( + "%s has no free port left between %d and %d, which is ten thousand of them — "+ + "something is assigning ports it never gives back", + node, firstAssignable, lastAssignable) + } + } + + if by, held := taken[machine]; held { + return Assigned{}, fmt.Errorf( + "%w: %s needs %d and %s already has it on %s. A port the protocol fixes can have one "+ + "holder per machine, so one of them has to go somewhere else", + ErrPortTaken, module, machine, by, node) + } + + _, err = i.store.Pool().Exec(ctx, + `insert into port_assignment (node, module, wanted, machine, fixed) + values ($1, $2, $3, $4, $5)`, + nodeID, module, wanted, machine, fixed) + if err != nil { + return Assigned{}, err + } + return Assigned{Module: module, Wanted: wanted, Machine: machine, Fixed: fixed}, nil +} + +/** Which machine ports are spoken for, and by whom. */ +func (i *Inventory) portsOn(ctx context.Context, nodeID any) (map[int]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select machine, module from port_assignment where node = $1`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[int]string{} + for rows.Next() { + var machine int + var module string + if err := rows.Scan(&machine, &module); err != nil { + return nil, err + } + out[machine] = module + } + return out, rows.Err() +} + +// PortsFor is every assignment a node holds, for composing its declaration. +func (i *Inventory) PortsFor(ctx context.Context, node string) ([]Assigned, error) { + record, err := i.NodeByName(ctx, node) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select module, wanted, machine, fixed from port_assignment + where node = $1 order by module, wanted`, record.ID) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Assigned + for rows.Next() { + var a Assigned + if err := rows.Scan(&a.Module, &a.Wanted, &a.Machine, &a.Fixed); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +// ReleasePorts gives back everything a module held on a machine, for when it is unassigned. +func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error { + record, err := i.NodeByName(ctx, node) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `delete from port_assignment where node = $1 and module = $2`, record.ID, module) + return err +} diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go new file mode 100644 index 0000000..dc057c5 --- /dev/null +++ b/internal/inventory/ports_test.go @@ -0,0 +1,151 @@ +package inventory + +import ( + "errors" + "testing" + + "github.com/novox/mesh-control/internal/catalogue" +) + +func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) { + t.Helper() + inv := fresh(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + for _, m := range modules { + if err := inv.RegisterModule(ctx, + catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + } + return inv, "anchor" +} + +// **Made once and kept.** A port that moved on every declaration would restart both ends each +// time, and would hand a consumer a number that was true when it was read. +func TestAPortIsAssignedOnceAndKept(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres") + first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + second, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if first.Machine != second.Machine { + t.Fatalf("asking twice moved the port: %d then %d", first.Machine, second.Machine) + } + if first.Machine == 5432 { + t.Error("the mesh handed back the port the module asked for, which is what it cannot know is free") + } +} + +// The fault this exists for: two modules wanting one number, which neither of them chose badly. +func TestTwoModulesWantingOnePortGetTwo(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "another-database") + a, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + b, err := inv.PortFor(t.Context(), node, "another-database", 5432, false) + if err != nil { + t.Fatal(err) + } + if a.Machine == b.Machine { + t.Fatalf("both were put on %d, which is the collision this exists to prevent", a.Machine) + } +} + +// A port the protocol fixes is used as written, because a mail system elsewhere is not a mail +// system. +func TestAFixedPortIsTheOneTheProtocolSays(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu") + got, err := inv.PortFor(t.Context(), node, "mailu", 25, true) + if err != nil { + t.Fatal(err) + } + if got.Machine != 25 { + t.Fatalf("mail was put on %d", got.Machine) + } + if !got.Fixed { + t.Error("it does not record that the protocol chose it, so nothing can refuse a second holder") + } +} + +// **A fixed port is a claim**: one holder per machine, refused by name at assignment rather than +// by a container runtime at apply. +func TestASecondModuleCannotHaveAFixedPort(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu", "other-mail") + if _, err := inv.PortFor(t.Context(), node, "mailu", 25, true); err != nil { + t.Fatal(err) + } + _, err := inv.PortFor(t.Context(), node, "other-mail", 25, true) + if err == nil { + t.Fatal("two modules were given port 25 on one machine") + } + if !errors.Is(err, ErrPortTaken) { + t.Errorf("the refusal is not the one a caller can recognise: %v", err) + } + if !contains(err.Error(), "mailu") { + t.Errorf("the refusal does not say who has it: %v", err) + } +} + +// An assigned port must not land on one the protocol fixed for something else. +func TestAnAssignedPortAvoidsAFixedOne(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu", "web") + fixed, err := inv.PortFor(t.Context(), node, "mailu", 20000, true) + if err != nil { + t.Fatal(err) + } + assigned, err := inv.PortFor(t.Context(), node, "web", 8080, false) + if err != nil { + t.Fatal(err) + } + if assigned.Machine == fixed.Machine { + t.Fatalf("an assignment landed on %d, which the protocol had fixed for something else", + fixed.Machine) + } +} + +// What a module gave back is available again. Otherwise a machine that ran a hundred modules over +// a year has a hundred ports it cannot explain. +func TestUnassigningGivesThePortBack(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres") + first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if err := inv.ReleasePorts(t.Context(), node, "postgres"); err != nil { + t.Fatal(err) + } + held, err := inv.PortsFor(t.Context(), node) + if err != nil { + t.Fatal(err) + } + if len(held) != 0 { + t.Fatalf("it still holds %v", held) + } + again, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if again.Machine != first.Machine { + t.Errorf("the freed port was not the first one offered again: %d then %d", + first.Machine, again.Machine) + } +} + +func contains(s, what string) bool { + return len(s) >= len(what) && (func() bool { + for i := 0; i+len(what) <= len(s); i++ { + if s[i:i+len(what)] == what { + return true + } + } + return false + })() +}