diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index e9f1579..be10c90 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -196,7 +196,11 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, // What that module says a consumer needs to know, with that node's settings on // it: a port somebody moved on the provider is a port its consumers must be told // about, and the two coming from different places is how they come to disagree. - serves := m.Serves[name] + assigned, err := portsOn(ctx, inv, o.node.Name, m.Module) + if err != nil { + return catalogue.World{}, err + } + serves := catalogue.ServedOn(m, name, assigned) if len(serves) > 0 { layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) if err != nil { @@ -253,6 +257,28 @@ func declarationWith(ctx context.Context, open *stores, node string, if err != nil { return nil, err } + // Where this machine puts what each module needs reachable (novox/hq ADR 0038). + // + // **Assigned here rather than written by a module**, because a module is written once and + // assigned anywhere: any number it picks is a guess about a machine it has never seen. Made + // before the declaration is composed, because the container's mapping, the rule set and what a + // consumer is told are all derived from it. + ports := map[string]map[int]int{} + for _, m := range plan.Modules { + for _, l := range m.Listens { + at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed) + if err != nil { + return nil, fmt.Errorf( + "%s needs %d reachable on %s and it could not be assigned: %w", + m.Module, l.Port, node, err) + } + if ports[m.Module] == nil { + ports[m.Module] = map[int]int{} + } + ports[m.Module][l.Port] = at.Machine + } + } + // And each module's own secrets — a superuser password, an administrator, an account. Made // per node, so a module running on three machines has three. needed := map[string]map[string]string{} @@ -302,7 +328,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } return plan.Declaration(catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names}) } @@ -562,3 +588,20 @@ func keyFor(ctx context.Context, open *stores, licence, node, module string) (st } return held.KeyFor(ctx, licence, node, module) } + +// portsOn is one module's assignments on one machine, by the port the software uses. +func portsOn( + ctx context.Context, inv *inventory.Inventory, node, module string, +) (map[int]int, error) { + all, err := inv.PortsFor(ctx, node) + if err != nil { + return nil, err + } + out := map[int]int{} + for _, a := range all { + if a.Module == module { + out[a.Wanted] = a.Machine + } + } + return out, nil +} diff --git a/examples/modules/dnsmasq.json b/examples/modules/dnsmasq.json index 7ed1ff6..79d311d 100644 --- a/examples/modules/dnsmasq.json +++ b/examples/modules/dnsmasq.json @@ -1,24 +1,50 @@ { "module": "dnsmasq", "version": "1", - - "requires": ["resolver-data"], - "provides": ["wildcard-resolution"], - "claims": [{"name": "the-dns-port", "scope": "node"}], - + "requires": [ + "resolver-data" + ], + "provides": [ + "wildcard-resolution" + ], + "claims": [ + { + "name": "the-dns-port", + "scope": "node" + } + ], "listens": [ - {"port": 53, "protocol": "udp", "from": "mesh", - "why": "names under every machine in this mesh, for this machine and what it runs"} + { + "port": 53, + "protocol": "udp", + "from": "mesh", + "why": "names under every machine in this mesh, for this machine and what it runs", + "fixed": true + } ], - "resources": [ - {"id": "package", "type": "package", "package": "dnsmasq"}, - - {"id": "config", "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it — including a container\n# on this machine — can ask.\n# 127.0.0.55 this machine's own use, for whatever points resolution at the\n# mesh. Not .53 or .54: systemd-resolved holds BOTH — .53 is its\n# stub and .54 its proxy stub — which this module asserted was\n# free until a machine said otherwise.\n#\n# Listening on a loopback address makes dnsmasq take the rest of\n# loopback with it, 127.0.0.1 included. That is why this module\n# claims `the-dns-port`: it takes the machine's DNS port, and\n# saying it takes only one address would be the same kind of\n# comfortable claim that .54 was free.\n#\n# .55 is a convention and not a reservation. If a future systemd\n# takes it, this line changes and nothing else does, which is the\n# reason it is written once here rather than in each module that\n# points at it.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.55\n\n# **It forwards nothing, and must not read resolv.conf to find out where to.**\n# Whatever points this machine at the mesh writes its own address into\n# resolv.conf — so a resolver that read it for upstreams would find itself,\n# and every query it could not answer locally would loop until its receive\n# queue filled. That is not theoretical: it filled with 15KB of queries and\n# every lookup on the machine hung.\n#\n# It needs no upstream because it is never asked for anything else: the\n# asking module routes only the mesh's suffix here and leaves the rest\n# wherever the machine already sent it.\nno-resolv\ndomain-needed\nbogus-priv\n"}, - - {"id": "service", "type": "service", "unit": "dnsmasq.service", - "state": "running", "boot": "enabled", - "restart-on": ["config", "mesh-resolver.nodes"]} + { + "id": "package", + "type": "package", + "package": "dnsmasq" + }, + { + "id": "config", + "type": "file", + "path": "/etc/dnsmasq.conf", + "mode": "0644", + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n\n# What the mesh computed: one wildcard per machine, its name and everything\n# under it. Rewritten whenever a machine joins or leaves, which is why the\n# service below reflects it.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Where it answers. Both are names the mesh chose, so this file needs to know\n# nothing about this particular machine:\n#\n# mesh0 the private network, so anything on it \u2014 including a container\n# on this machine \u2014 can ask.\n# 127.0.0.55 this machine's own use, for whatever points resolution at the\n# mesh. Not .53 or .54: systemd-resolved holds BOTH \u2014 .53 is its\n# stub and .54 its proxy stub \u2014 which this module asserted was\n# free until a machine said otherwise.\n#\n# Listening on a loopback address makes dnsmasq take the rest of\n# loopback with it, 127.0.0.1 included. That is why this module\n# claims `the-dns-port`: it takes the machine's DNS port, and\n# saying it takes only one address would be the same kind of\n# comfortable claim that .54 was free.\n#\n# .55 is a convention and not a reservation. If a future systemd\n# takes it, this line changes and nothing else does, which is the\n# reason it is written once here rather than in each module that\n# points at it.\n#\n# bind-dynamic rather than bind-interfaces: mesh0 does not exist until the\n# machine is on the private network, and binding an interface that is not there\n# yet fails to start rather than waiting for it.\nbind-dynamic\ninterface=mesh0\nlisten-address=127.0.0.55\n\n# **It forwards nothing, and must not read resolv.conf to find out where to.**\n# Whatever points this machine at the mesh writes its own address into\n# resolv.conf \u2014 so a resolver that read it for upstreams would find itself,\n# and every query it could not answer locally would loop until its receive\n# queue filled. That is not theoretical: it filled with 15KB of queries and\n# every lookup on the machine hung.\n#\n# It needs no upstream because it is never asked for anything else: the\n# asking module routes only the mesh's suffix here and leaves the rest\n# wherever the machine already sent it.\nno-resolv\ndomain-needed\nbogus-priv\n" + }, + { + "id": "service", + "type": "service", + "unit": "dnsmasq.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "config", + "mesh-resolver.nodes" + ] + } ] } diff --git a/examples/modules/gitea.json b/examples/modules/gitea.json index 987add6..c307925 100644 --- a/examples/modules/gitea.json +++ b/examples/modules/gitea.json @@ -69,7 +69,7 @@ "/var/lib/gitea/server.env" ], "ports": [ - "3000:3000", + "3000", "2222:22" ], "volumes": [ diff --git a/examples/modules/keycloak.json b/examples/modules/keycloak.json index 530dbf0..5096b20 100644 --- a/examples/modules/keycloak.json +++ b/examples/modules/keycloak.json @@ -74,7 +74,7 @@ "/var/lib/keycloak/database.env" ], "ports": [ - "8080:8080" + "8080" ] } ] diff --git a/examples/modules/mailu.json b/examples/modules/mailu.json index c0d7a2c..b1fe323 100644 --- a/examples/modules/mailu.json +++ b/examples/modules/mailu.json @@ -9,25 +9,29 @@ "port": 25, "protocol": "tcp", "from": "anywhere", - "why": "mail from other mail servers" + "why": "mail from other mail servers", + "fixed": true }, { "port": 465, "protocol": "tcp", "from": "anywhere", - "why": "submission over TLS" + "why": "submission over TLS", + "fixed": true }, { "port": 587, "protocol": "tcp", "from": "anywhere", - "why": "submission" + "why": "submission", + "fixed": true }, { "port": 993, "protocol": "tcp", "from": "anywhere", - "why": "IMAP over TLS" + "why": "IMAP over TLS", + "fixed": true }, { "port": 7080, @@ -255,10 +259,10 @@ "/var/lib/mailu/secret.env" ], "ports": [ - "25:25", - "465:465", - "587:587", - "993:993", + "25", + "465", + "587", + "993", "7080:80" ], "volumes": [ diff --git a/examples/modules/minio.json b/examples/modules/minio.json index 1feb6a5..12c285a 100644 --- a/examples/modules/minio.json +++ b/examples/modules/minio.json @@ -20,7 +20,6 @@ ], "serves": { "s3-bucket": { - "port": 9000, "scheme": "http", "region": "us-east-1" } @@ -81,7 +80,7 @@ "/var/lib/minio/root.env" ], "ports": [ - "9000:9000" + "9000" ], "volumes": [ "/services/minio/data/data1-1:/data" diff --git a/examples/modules/modules_test.go b/examples/modules/modules_test.go index cbd1a2a..c152f61 100644 --- a/examples/modules/modules_test.go +++ b/examples/modules/modules_test.go @@ -429,10 +429,13 @@ func declareOnItsOwn(t *testing.T, shelf map[string]catalogue.Manifest, // providing example says it serves. offered := map[string][]catalogue.Provider{} for _, want := range m.Requires { + // Built the way the control plane builds it: what a provider tells a consumer includes + // the port, and the module no longer writes that into `serves` by hand — it says it once + // in `listens` and the mesh puts it there (novox/hq ADR 0038). serves := map[string]any{} for _, other := range shelf { - if s, said := other.Serves[want]; said { - serves = s + if _, said := other.Serves[want]; said { + serves = catalogue.ServedOn(other, want, nil) } } offered[want] = []catalogue.Provider{ diff --git a/examples/modules/object-store.json b/examples/modules/object-store.json index 22243b8..5d2d3df 100644 --- a/examples/modules/object-store.json +++ b/examples/modules/object-store.json @@ -20,7 +20,6 @@ ], "serves": { "s3-bucket": { - "port": 9000, "scheme": "http", "region": "us-east-1" } @@ -60,7 +59,7 @@ "MINIO_ROOT_USER": "meshroot" }, "ports": [ - "9000:9000" + "9000" ], "volumes": [ "mesh-store-data:/data", diff --git a/examples/modules/postgres.json b/examples/modules/postgres.json index 7bd4edb..e2d8f03 100644 --- a/examples/modules/postgres.json +++ b/examples/modules/postgres.json @@ -19,9 +19,7 @@ } ], "serves": { - "postgres-database": { - "port": 5432 - } + "postgres-database": {} }, "receives": { "postgres-database": "/var/lib/postgres/grants/mesh.json" @@ -77,7 +75,7 @@ "/var/lib/postgres/superuser.env" ], "ports": [ - "5432:5432" + "5432" ], "volumes": [ "/services/postgres/db-data:/var/lib/postgresql/data" diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 7395288..ea5d8ed 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "strings" ) @@ -97,6 +98,24 @@ type Rendering struct { // rather than resolved, because who consumes a node is a fact about the rest of the mesh and // resolution answers questions about one machine. Grants []Grant + + // Ports is where this machine puts what each module needs reachable, by module and by the + // port the software itself uses (novox/hq ADR 0038). + // + // **The one place the number now lives.** A module used to write it three times — for the rule + // set, for what a consumer is told, and for what the runtime publishes — and nothing checked + // that the three agreed. They are all derived from this. + Ports map[string]map[int]int +} + +// machinePort is where a module's port lives on this machine, or the port itself when the mesh has +// not been asked. Unassigned is not an error here: a module with no `listens` never needed one, +// and a caller composing a declaration without a store still gets something coherent. +func (r Rendering) machinePort(module string, wanted int) int { + if at, known := r.Ports[module][wanted]; known { + return at + } + return wanted } // Declaration is everything the resolved modules put on the node, with settings applied. @@ -120,7 +139,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // Once, from every module's listens -- not per module. A module receiving only its own ports // would write a rule set that closed every other module on the machine. - rules, err := r.Filtering(with.Generators) + rules, err := r.Filtering(with.Generators, with.Ports) if err != nil { return nil, err } @@ -345,6 +364,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if err := pinned(copied, m.Module); err != nil { return nil, err } + publishedOn(copied, m.Module, with) copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those // are prefixed too or they would point at nothing. @@ -593,7 +613,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti // nothing resolves would be worse than naming it by an address that always works. func here(r Resolution, requirement string) *Needed { for _, m := range r.Modules { - serves, said := m.Serves[requirement] + _, said := m.Serves[requirement] + serves := ServedOn(m, requirement, nil) if !said || len(serves) == 0 { continue } @@ -667,3 +688,90 @@ func pinned(resource map[string]any, module string) error { "image — it would be fetched and fail there. Resolve the tag to a digest first", module, resource["id"]) } + +// publishedOn puts the machine's own port on the outside of a container's mapping. +// +// **A module writes the port the software uses; the mesh says where the machine puts it** +// (novox/hq ADR 0038). So `"5432"` means *publish what the software calls 5432*, and this fills in +// the half only the mesh can know. +// +// A mapping written the long way — `"5433:5432"` — is left exactly as it is. Some things genuinely +// must be pinned down by hand, and quietly overruling somebody who wrote both halves would be +// worse than not offering the short form at all. +func publishedOn(resource map[string]any, module string, with Rendering) { + if fmt.Sprint(resource["type"]) != "container" { + return + } + listed, ok := resource["ports"].([]any) + if !ok { + return + } + out := make([]any, 0, len(listed)) + for _, entry := range listed { + written := fmt.Sprint(entry) + if strings.Contains(written, ":") { + out = append(out, written) + continue + } + wanted, err := strconv.Atoi(strings.TrimSpace(written)) + if err != nil { + // Not a port at all. Passed through, so the host refuses it with its own words rather + // than this quietly dropping something somebody meant. + out = append(out, written) + continue + } + out = append(out, fmt.Sprintf("%d:%d", with.machinePort(module, wanted), wanted)) + } + resource["ports"] = out +} + +// ServedOn is what a provider tells a consumer, with the port that machine actually uses. +// +// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three +// times — for the rule set, for what a consumer is told, and for what the runtime publishes — and +// nothing checked that the three agreed. This is what fills the second in. +// +// The assignment wins when there is one, and the declared port stands in when there is not: a +// caller composing without a store still gets something coherent, and a mesh that has assigned one +// tells the truth about where it put it. +// +// Only when the module offers exactly one port. A module offering several has not said which +// belongs to which provision, and guessing would give a consumer a port that answers something +// else — so it keeps whatever the manifest said, which may be nothing. +func ServedOn(m Manifest, provision string, ports map[int]int) map[string]any { + serves := m.Serves[provision] + + out := make(map[string]any, len(serves)+1) + for k, v := range serves { + out[k] = v + } + if _, said := out["port"]; said { + // Written by hand. Redirected to wherever the machine put it, and otherwise left alone. + if number, ok := asPort(out["port"]); ok { + if at, known := ports[number]; known { + out["port"] = at + } + } + return out + } + if len(m.Listens) != 1 { + return out + } + wanted := m.Listens[0].Port + if at, known := ports[wanted]; known { + out["port"] = at + } else { + out["port"] = wanted + } + return out +} + +func asPort(v any) (int, bool) { + switch n := v.(type) { + case int: + return n, true + case float64: + return int(n), true + } + return 0, false +} diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index b224665..36a1aba 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -31,7 +31,7 @@ type Rule struct { // a consequence of what runs on it, not a second list kept in step by hand. Nothing else opens a // port: **what is not declared is closed**, which is the property that makes the derivation worth // having rather than merely tidy. -func (r Resolution) Filtering(computed map[string]Generator) ([]Rule, error) { +func (r Resolution) Filtering(computed map[string]Generator, ports map[string]map[int]int) ([]Rule, error) { // Keyed by what actually distinguishes an opening. Two modules wanting :443 from the mesh is // one rule with two sources; one wanting it from the mesh and another from anywhere is two, // and they are collapsed below -- deliberately, and only in the widening direction. @@ -62,10 +62,18 @@ func (r Resolution) Filtering(computed map[string]Generator) ([]Rule, error) { } } for _, l := range opens { - at := opening{port: l.Port, protocol: l.At(), from: l.From} + // **The port the machine actually publishes on** (novox/hq ADR 0038). A module says + // the port its software uses; the mesh chooses where the machine puts it. A rule + // naming the first would open a port nothing listens on and leave the real one shut, + // which is a firewall that reports success and blocks the service. + port := l.Port + if at, known := ports[m.Module][l.Port]; known { + port = at + } + at := opening{port: port, protocol: l.At(), from: l.From} rule, seen := found[at] if !seen { - rule = &Rule{Port: l.Port, Protocol: l.At(), From: l.From} + rule = &Rule{Port: port, Protocol: l.At(), From: l.From} found[at] = rule } rule.Because = append(rule.Because, m.Module) diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index 1d9669d..204e37b 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -369,7 +369,7 @@ func TestAMachineOffTheNetworkIsBoundToItselfByAnAddressThatWorks(t *testing.T) // mustFilter is the rule set, refusing to continue if it could not be computed. func mustFilter(t *testing.T, r Resolution, computed map[string]Generator) []Rule { t.Helper() - rules, err := r.Filtering(computed) + rules, err := r.Filtering(computed, nil) if err != nil { t.Fatalf("no rule set could be computed: %v", err) } @@ -440,7 +440,7 @@ func TestAComputedModulesOwnListensAreNotLost(t *testing.T) { func TestAGeneratorThatCannotSayRefusesTheRuleSet(t *testing.T) { _, err := Resolution{Node: "anchor", Modules: []Manifest{{Module: "networking", Computed: "mesh-network"}}, - }.Filtering(map[string]Generator{"mesh-network": cannotSay{}}) + }.Filtering(map[string]Generator{"mesh-network": cannotSay{}}, nil) if err == nil { t.Fatal("a machine whose open ports could not be computed was given a rule set anyway") } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index a1f4505..c151e00 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -318,6 +318,16 @@ type Listening struct { // From is who may reach it. Required, because a rule with no source is open and must say so // rather than appear to restrict something. From string `json:"from"` + // Fixed means the protocol chose this number, so the machine must use it too. + // + // **The exception, and it is a real one** (novox/hq ADR 0038). Mail is 25, submission is 587, + // IMAP over TLS is 993 — a mail system on a port the mesh picked is a mail system nothing can + // deliver to. Everything else the mesh assigns, because a module cannot know what else is on + // the machine it lands on. + // + // A fixed port is a **claim**: one holder per machine, and the second is refused by name when + // it is assigned rather than by a container runtime when it is applied. + Fixed bool `json:"fixed,omitempty"` // Why this port is open, for somebody reading a generated rule set and wondering. Why string `json:"why,omitempty"` } diff --git a/internal/catalogue/resolve.go b/internal/catalogue/resolve.go index a28eda2..13288b4 100644 --- a/internal/catalogue/resolve.go +++ b/internal/catalogue/resolve.go @@ -474,8 +474,10 @@ func servedHere(catalogue map[string]Manifest, chosen map[string]bool, want stri if !chosen[name] { continue } - if serves, ok := m.Serves[want]; ok { - return serves + if _, ok := m.Serves[want]; ok { + // Without assignments: this is resolution, which runs before a machine's ports are + // known. The declaration fills the machine's own in afterwards, where it has them. + return ServedOn(m, want, nil) } } return nil diff --git a/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql b/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql new file mode 100644 index 0000000..ac966d5 --- /dev/null +++ b/internal/inventory/migrations/0016-the-mesh-assigns-the-port.sql @@ -0,0 +1,33 @@ +-- Which port a machine uses for what a module needs reachable. +-- +-- novox/hq ADR 0038. A module cannot choose this: it is written once and assigned anywhere, so any +-- number it picks is a guess about a machine it has never seen. Two modules guessing the same one +-- is not a mistake either of them made -- it is a database module meeting the mesh's own store and +-- being told, by a container runtime three layers down, that the port is already allocated. +-- +-- **Made once and kept**, exactly as a credential is. A port that moved on every declaration would +-- restart both ends each time, and would hand a consumer a number that was true when it was read. + +create table port_assignment ( + node uuid not null references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + + -- The port the software itself uses -- what a module writes down, and the only part it knows. + wanted integer not null, + -- What the machine publishes it on. The same as `wanted` when the protocol fixes it. + machine integer not null, + + -- Whether the protocol fixed it. Kept rather than derived: *this is 25 because it must be* + -- and *this is 25 because it was free* are different facts, and only the first refuses a + -- second holder. + fixed boolean not null default false, + + assigned_at timestamptz not null default now(), + + primary key (node, module, wanted), + + -- **One machine port, one holder.** The constraint is the point: a second module cannot be + -- given a port the first has, and finding that out here is finding it out at assignment + -- rather than at apply. + unique (node, machine) +); diff --git a/internal/inventory/ports.go b/internal/inventory/ports.go new file mode 100644 index 0000000..52d8b23 --- /dev/null +++ b/internal/inventory/ports.go @@ -0,0 +1,186 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// Which port a machine uses for what a module needs reachable. +// +// **A module cannot choose this** (novox/hq ADR 0038). It is written once and assigned anywhere, +// so any number it picks is a guess about a machine it has never seen. The mesh is the only thing +// that knows what else is there, so the mesh chooses — and a module says only that something must +// be reachable, and what the software itself calls it. + +// Assigned is where the machine puts one of a module's ports. +type Assigned struct { + Module string + // Wanted is the port the software uses — what the module wrote down. + Wanted int + // Machine is where this machine publishes it. + Machine int + // Fixed means the protocol chose it, not the mesh. + Fixed bool +} + +// The range the mesh assigns from. +// +// High and unprivileged, so an assignment never needs root and never lands on something a person +// would recognise. Below the range Linux uses for outgoing connections, so an assignment cannot +// collide with a port the kernel handed to something else while the machine was working. +const ( + firstAssignable = 20000 + lastAssignable = 29999 +) + +// ErrPortTaken is returned when a port the protocol fixes is already held by another module. +var ErrPortTaken = errors.New("that port is already held on this machine") + +// PortFor is where one module's port lives on one machine, choosing it the first time. +// +// **Kept once chosen.** A port that moved on every declaration would restart both ends each time, +// and would hand a consumer a number that was true when it was read — which is the same argument +// that makes a credential stable. +func (i *Inventory) PortFor( + ctx context.Context, node, module string, wanted int, fixed bool, +) (Assigned, error) { + record, err := i.NodeByName(ctx, node) + if err != nil { + return Assigned{}, err + } + + var held Assigned + err = i.store.Pool().QueryRow(ctx, + `select machine, fixed from port_assignment + where node = $1 and module = $2 and wanted = $3`, + record.ID, module, wanted).Scan(&held.Machine, &held.Fixed) + if err == nil { + held.Module, held.Wanted = module, wanted + // A module that has become fixed since it was assigned must move to the port the protocol + // requires. Said rather than silently kept: a mail system on the port it was lent is a + // mail system nothing can deliver to. + if fixed && held.Machine != wanted { + if err := i.freePort(ctx, record.ID, module, wanted); err != nil { + return Assigned{}, err + } + return i.assignPort(ctx, record.ID, node, module, wanted, true) + } + return held, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return Assigned{}, err + } + return i.assignPort(ctx, record.ID, node, module, wanted, fixed) +} + +func (i *Inventory) freePort(ctx context.Context, node any, module string, wanted int) error { + _, err := i.store.Pool().Exec(ctx, + `delete from port_assignment where node = $1 and module = $2 and wanted = $3`, + node, module, wanted) + return err +} + +func (i *Inventory) assignPort( + ctx context.Context, nodeID any, node, module string, wanted int, fixed bool, +) (Assigned, error) { + taken, err := i.portsOn(ctx, nodeID) + if err != nil { + return Assigned{}, err + } + + machine := wanted + if !fixed { + // The lowest free one, so a machine's assignments are stable and readable rather than + // scattered — and so the same set of modules on two machines gets the same numbers, which + // makes a difference between two machines mean something. + machine = 0 + for candidate := firstAssignable; candidate <= lastAssignable; candidate++ { + if _, held := taken[candidate]; !held { + machine = candidate + break + } + } + if machine == 0 { + return Assigned{}, fmt.Errorf( + "%s has no free port left between %d and %d, which is ten thousand of them — "+ + "something is assigning ports it never gives back", + node, firstAssignable, lastAssignable) + } + } + + if by, held := taken[machine]; held { + return Assigned{}, fmt.Errorf( + "%w: %s needs %d and %s already has it on %s. A port the protocol fixes can have one "+ + "holder per machine, so one of them has to go somewhere else", + ErrPortTaken, module, machine, by, node) + } + + _, err = i.store.Pool().Exec(ctx, + `insert into port_assignment (node, module, wanted, machine, fixed) + values ($1, $2, $3, $4, $5)`, + nodeID, module, wanted, machine, fixed) + if err != nil { + return Assigned{}, err + } + return Assigned{Module: module, Wanted: wanted, Machine: machine, Fixed: fixed}, nil +} + +/** Which machine ports are spoken for, and by whom. */ +func (i *Inventory) portsOn(ctx context.Context, nodeID any) (map[int]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select machine, module from port_assignment where node = $1`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[int]string{} + for rows.Next() { + var machine int + var module string + if err := rows.Scan(&machine, &module); err != nil { + return nil, err + } + out[machine] = module + } + return out, rows.Err() +} + +// PortsFor is every assignment a node holds, for composing its declaration. +func (i *Inventory) PortsFor(ctx context.Context, node string) ([]Assigned, error) { + record, err := i.NodeByName(ctx, node) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select module, wanted, machine, fixed from port_assignment + where node = $1 order by module, wanted`, record.ID) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Assigned + for rows.Next() { + var a Assigned + if err := rows.Scan(&a.Module, &a.Wanted, &a.Machine, &a.Fixed); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +// ReleasePorts gives back everything a module held on a machine, for when it is unassigned. +func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error { + record, err := i.NodeByName(ctx, node) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `delete from port_assignment where node = $1 and module = $2`, record.ID, module) + return err +} diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go new file mode 100644 index 0000000..dc057c5 --- /dev/null +++ b/internal/inventory/ports_test.go @@ -0,0 +1,151 @@ +package inventory + +import ( + "errors" + "testing" + + "github.com/novox/mesh-control/internal/catalogue" +) + +func aNodeWithModules(t *testing.T, modules ...string) (*Inventory, string) { + t.Helper() + inv := fresh(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + for _, m := range modules { + if err := inv.RegisterModule(ctx, + catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + } + return inv, "anchor" +} + +// **Made once and kept.** A port that moved on every declaration would restart both ends each +// time, and would hand a consumer a number that was true when it was read. +func TestAPortIsAssignedOnceAndKept(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres") + first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + second, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if first.Machine != second.Machine { + t.Fatalf("asking twice moved the port: %d then %d", first.Machine, second.Machine) + } + if first.Machine == 5432 { + t.Error("the mesh handed back the port the module asked for, which is what it cannot know is free") + } +} + +// The fault this exists for: two modules wanting one number, which neither of them chose badly. +func TestTwoModulesWantingOnePortGetTwo(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "another-database") + a, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + b, err := inv.PortFor(t.Context(), node, "another-database", 5432, false) + if err != nil { + t.Fatal(err) + } + if a.Machine == b.Machine { + t.Fatalf("both were put on %d, which is the collision this exists to prevent", a.Machine) + } +} + +// A port the protocol fixes is used as written, because a mail system elsewhere is not a mail +// system. +func TestAFixedPortIsTheOneTheProtocolSays(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu") + got, err := inv.PortFor(t.Context(), node, "mailu", 25, true) + if err != nil { + t.Fatal(err) + } + if got.Machine != 25 { + t.Fatalf("mail was put on %d", got.Machine) + } + if !got.Fixed { + t.Error("it does not record that the protocol chose it, so nothing can refuse a second holder") + } +} + +// **A fixed port is a claim**: one holder per machine, refused by name at assignment rather than +// by a container runtime at apply. +func TestASecondModuleCannotHaveAFixedPort(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu", "other-mail") + if _, err := inv.PortFor(t.Context(), node, "mailu", 25, true); err != nil { + t.Fatal(err) + } + _, err := inv.PortFor(t.Context(), node, "other-mail", 25, true) + if err == nil { + t.Fatal("two modules were given port 25 on one machine") + } + if !errors.Is(err, ErrPortTaken) { + t.Errorf("the refusal is not the one a caller can recognise: %v", err) + } + if !contains(err.Error(), "mailu") { + t.Errorf("the refusal does not say who has it: %v", err) + } +} + +// An assigned port must not land on one the protocol fixed for something else. +func TestAnAssignedPortAvoidsAFixedOne(t *testing.T) { + inv, node := aNodeWithModules(t, "mailu", "web") + fixed, err := inv.PortFor(t.Context(), node, "mailu", 20000, true) + if err != nil { + t.Fatal(err) + } + assigned, err := inv.PortFor(t.Context(), node, "web", 8080, false) + if err != nil { + t.Fatal(err) + } + if assigned.Machine == fixed.Machine { + t.Fatalf("an assignment landed on %d, which the protocol had fixed for something else", + fixed.Machine) + } +} + +// What a module gave back is available again. Otherwise a machine that ran a hundred modules over +// a year has a hundred ports it cannot explain. +func TestUnassigningGivesThePortBack(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres") + first, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if err := inv.ReleasePorts(t.Context(), node, "postgres"); err != nil { + t.Fatal(err) + } + held, err := inv.PortsFor(t.Context(), node) + if err != nil { + t.Fatal(err) + } + if len(held) != 0 { + t.Fatalf("it still holds %v", held) + } + again, err := inv.PortFor(t.Context(), node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if again.Machine != first.Machine { + t.Errorf("the freed port was not the first one offered again: %d then %d", + first.Machine, again.Machine) + } +} + +func contains(s, what string) bool { + return len(s) >= len(what) && (func() bool { + for i := 0; i+len(what) <= len(s); i++ { + if s[i:i+len(what)] == what { + return true + } + } + return false + })() +}