From dbf62b5212096643c5528e78f9a01d7a78556f6e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:31:07 +0200 Subject: [PATCH] Read the foundation's ports from each node's settings, wherever a port is used (hq ADR 0100) --- cmd/mesh-controller/plan.go | 37 +++++++++-- cmd/mesh-controller/sendable_test.go | 57 +++++++++++++++++ internal/catalogue/adoption_test.go | 60 ++++++++++++++++++ internal/catalogue/declaration.go | 37 ++++++++++- internal/catalogue/filtering.go | 92 ++++++++++++++++++++++++++++ internal/catalogue/settings.go | 10 +++ internal/inventory/catalogue.go | 2 +- internal/inventory/ports.go | 42 +++++++++++++ internal/inventory/ports_test.go | 21 +++++++ 9 files changed, 352 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 919817f..48ff6d9 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -233,12 +233,19 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, if err != nil { return catalogue.World{}, err } + layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) + if err != nil { + return catalogue.World{}, err + } + // A port that node was given is where its consumers reach it (novox/hq ADR + // 0100). Unreadable given ports are that node's refusal to report, not this one's. + if given, err := catalogue.GivenPorts(m, layers); err == nil { + for wanted, at := range given { + assigned[wanted] = at + } + } serves := catalogue.ServedOn(m, name, assigned) if len(serves) > 0 { - layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) - if err != nil { - return catalogue.World{}, err - } serves, err = catalogue.Settle(serves, layers) if err != nil { return catalogue.World{}, err @@ -364,9 +371,30 @@ func renderingFor(ctx context.Context, open *stores, node string, } } + // The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's + // ports, as genesis chose them. A given port wins over anything assigned and over a manifest's + // own long-form mapping. + given := map[string]map[int]int{} + for _, m := range plan.Modules { + g, err := catalogue.GivenPorts(m, settings[m.Module]) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if g != nil { + given[m.Module] = g + } + } + ports := map[string]map[int]int{} for _, m := range plan.Modules { for _, l := range m.Listens { + if at, isGiven := given[m.Module][l.Port]; isGiven { + if ports[m.Module] == nil { + ports[m.Module] = map[int]int{} + } + ports[m.Module][l.Port] = at + continue + } // **Only a port the module actually publishes is the mesh's to move.** A container's // mapping is the thing that translates; without one the software binds what it binds, // and an assignment would not move the service — it would open the wrong number in the @@ -515,6 +543,7 @@ func renderingFor(ctx context.Context, open *stores, node string, Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, + Given: given, }, record, nil } diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index bf052b9..cb82110 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -168,3 +168,60 @@ func stdoutOf(t *testing.T, run func() error) string { } return out } + +// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other +// machines are told to reach it, and a port given for the whole mesh is refused. +func TestConsumersAreToldTheGivenPort(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { + t.Fatal(err) + } + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "database" { + told = n.Serves["port"] + } + } + if told != 5433 { + t.Fatalf("the consumer is told the store is on %v", told) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { + t.Fatalf("the store publishes %v", r["ports"]) + } + } + + if err := open.inventory.SetSettings(ctx, "", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := declarationFor(ctx, open, "anchor", plan, settings); err == nil || + !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was not refused: %v", err) + } +} diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 20569bd..c5446b8 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -221,3 +221,63 @@ func TestAGuardedPortMustBeAPort(t *testing.T) { func keys[V any](m map[string]V) []string { return sortedKeys(m) } + +// novox/hq ADR 0100: the foundation's ports are the node's. Given 5433 for the store, every place +// that uses the port reads it from there: the container, the filter, the openings, the guard. +func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) { + given := map[string]map[int]int{"postgres": {5432: 5433}, "lavinmq": {15672: 15673}} + for _, adopted := range []bool{true, false} { + with := anchorRendering(adopted) + with.Given = given + with.Ports["postgres"] = map[int]int{5432: 5433} + composed, err := anAdoptedAnchor().Compose(with) + if err != nil { + t.Fatal(err) + } + got := byID(composed.Resources) + if ports := got["postgres.server"]["ports"]; !reflect.DeepEqual(ports, []any{"5433:5432"}) { + t.Fatalf("the store's container publishes %v", ports) + } + if ports := got["lavinmq.server"]["ports"]; !reflect.DeepEqual(ports, + []any{"5671:5671", "5672:5672", "127.0.0.1:15673:15672"}) { + t.Fatalf("the broker's container publishes %v", ports) + } + if !adopted { + filter, _ := got["nftables.filtering"]["content"].(string) + if !strings.Contains(filter, "tcp dport 5433 accept") || strings.Contains(filter, "5432") { + t.Fatalf("the filter does not use the given port:\n%s", filter) + } + continue + } + if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 { + t.Fatalf("no opening for the given port: %v", keys(got)) + } + if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 15673}) { + t.Fatalf("the guard does not guard the given ports:\n%s", guard) + } + } +} + +func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) { + store := anAdoptedAnchor().Modules[1] + node := func(v any) []Layer { + return []Layer{{From: "anchor", Values: map[string]any{PortsSetting: v}}} + } + if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil || + got[5432] != 5433 { + t.Fatalf("a node's given port was not read: %v %v", got, err) + } + if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer, + Values: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil { + t.Fatal("a port given for the whole mesh was accepted") + } + if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil { + t.Fatal("a port the module neither listens on, publishes nor guards was given") + } + if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil { + t.Fatal("a machine port that is not a port was given") + } + if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 { + t.Fatalf("a given port is called stray: %v", stray) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 59faa42..2e07ddf 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -133,12 +133,20 @@ type Rendering struct { // force, so no module that loads a filter is declared there, and what the mesh needs // reachable is declared as openings, with its own ports guarded by a table that only refuses. Adopted bool + + // Given is the machine ports this node was given for its modules' ports, by module and by the + // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. + // They win over anything the mesh would assign and over a manifest's own long-form mapping. + Given map[string]map[int]int } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has // not been asked. Unassigned is not an error here: a module with no `listens` never needed one, // and a caller composing a declaration without a store still gets something coherent. func (r Rendering) machinePort(module string, wanted int) int { + if at, given := r.Given[module][wanted]; given { + return at + } if at, known := r.Ports[module][wanted]; known { return at } @@ -1188,7 +1196,11 @@ func publishedOn(resource map[string]any, module string, with Rendering) { for _, entry := range listed { written := fmt.Sprint(entry) if strings.Contains(written, ":") { - out = append(out, written) + // Written the long way, and left alone — unless this node was given a machine port for + // it (novox/hq ADR 0100): the foundation's ports are the node's, and a manifest's + // number is only the default. The outer port only; an address and the software's + // port stay as written. + out = append(out, givenOuter(written, with.Given[module])) continue } wanted, err := strconv.Atoi(strings.TrimSpace(written)) @@ -1203,6 +1215,29 @@ func publishedOn(resource map[string]any, module string, with Rendering) { resource["ports"] = out } +// givenOuter rewrites the machine side of a long-form mapping to the port this node was given for +// its software side, when it was given one. +func givenOuter(written string, given map[int]int) string { + if len(given) == 0 { + return written + } + mapping, protocol := written, "" + if cut := strings.LastIndex(written, "/"); cut >= 0 { + mapping, protocol = written[:cut], written[cut:] + } + parts := strings.Split(mapping, ":") + inner, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1])) + if err != nil { + return written + } + at, ok := given[inner] + if !ok { + return written + } + parts[len(parts)-2] = strconv.Itoa(at) + return strings.Join(parts, ":") + protocol +} + // ServedOn is what a provider tells a consumer, with the port that machine actually uses. // // **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index 3726e87..ae1354f 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -457,3 +457,95 @@ func byFamily(addresses []string) (four []string, six []string) { } return four, six } + +// PortsSetting is the settings key that gives a module's port a machine port on one node (novox/hq +// ADR 0100): +// +// {"ports": {"5432": 5433}} +// +// puts what the software calls 5432 on the machine's 5433. The foundation's ports are the node's: +// every one is an input to genesis, checked free there, and becomes that node's setting for the +// foundation's modules — the catalogue's numbers are only their defaults. Keyed by the port the +// software uses, like expose; the value is where the machine puts it. +const PortsSetting = "ports" + +// MeshWideLayer is what a layer set for the whole mesh is called, rather than for one node. +const MeshWideLayer = "the mesh" + +// GivenPorts reads a module's given machine ports from its settings: software port → machine port. +// +// Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every +// machine is the collision this exists to avoid — and for a port the module neither listens on, +// publishes from a container, nor guards: a given port that reaches nothing is a setting somebody +// believes changed something. +func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) { + known := map[int]bool{} + for _, l := range m.Listens { + known[l.Port] = true + } + for _, p := range m.Guards { + known[p] = true + } + for _, p := range containerPorts(m) { + known[p] = true + } + out := map[int]int{} + for _, layer := range layers { + raw, ok := layer.Values[PortsSetting] + if !ok { + continue + } + if layer.From == MeshWideLayer { + return nil, fmt.Errorf("%s: %s is given per node — a port is a fact about one "+ + "machine; set it with --node", m.Module, PortsSetting) + } + entries, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: %s is a { port: machine-port } map, and %q set it to "+ + "something else", m.Module, PortsSetting, layer.From) + } + for portText, value := range entries { + port, err := strconv.Atoi(portText) + if err != nil { + return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText) + } + if !known[port] { + return nil, fmt.Errorf("%s gives port %d a machine port, and it neither listens "+ + "on, publishes nor guards %d — the setting reaches nothing", m.Module, port, port) + } + at, ok := asPort(value) + if !ok || at < 1 || at > 65535 { + return nil, fmt.Errorf("%s gives port %d the machine port %v, which is not a port", + m.Module, port, value) + } + out[port] = at + } + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +// containerPorts are the software ports a module's containers publish, whichever form they are +// written in. +func containerPorts(m Manifest) []int { + var out []int + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + listed, _ := r["ports"].([]any) + for _, entry := range listed { + written := strings.TrimSpace(fmt.Sprint(entry)) + if cut := strings.LastIndex(written, "/"); cut >= 0 { + written = written[:cut] + } + parts := strings.Split(written, ":") + if n, err := strconv.Atoi(parts[len(parts)-1]); err == nil { + out = append(out, n) + } + } + } + return out +} diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 5fa4865..854a9cf 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -101,6 +101,11 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what merged := deepCopy(base) for _, layer := range layers { for key, value := range layer.Values { + if key == PortsSetting { + // Where the machine puts a port is the mesh's to apply, not a value for a file or + // for what a consumer is told (novox/hq ADR 0100); it reaches both as the port. + continue + } if protected[key] { // The module said it must own this one. Refused rather than ignored: a setting // that is quietly dropped is somebody believing they changed something. @@ -176,6 +181,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string { if key == ExposeSetting && len(m.Listens) > 0 { continue } + // `ports` gives a module's port a machine port on one node (novox/hq ADR 0100), + // validated in GivenPorts, so it is not stray here either. + if key == PortsSetting { + continue + } unused = append(unused, fmt.Sprintf( "%s sets %q, and %s has no file or contribution to merge it into", layer.From, key, m.Module)) diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 71527d3..322de46 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -654,7 +654,7 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ } from := nodeName if meshWide { - from = "the mesh" + from = catalogue.MeshWideLayer } layers = append(layers, catalogue.Layer{From: from, Values: values}) } diff --git a/internal/inventory/ports.go b/internal/inventory/ports.go index 3e031c4..20cb4ed 100644 --- a/internal/inventory/ports.go +++ b/internal/inventory/ports.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "errors" "fmt" @@ -132,6 +133,17 @@ func (i *Inventory) assignPort( taken[port] = "something this machine already runs" } } + // And every port this machine was given for a module (novox/hq ADR 0100): the foundation's + // ports, as genesis chose them, are the node's settings and never the mesh's to hand out. + given, err := i.givenOn(ctx, nodeID) + if err != nil { + return Assigned{}, err + } + for port, by := range given { + if _, mine := taken[port]; !mine { + taken[port] = by + } + } machine := wanted if !fixed { @@ -258,3 +270,33 @@ func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error `delete from port_assignment where node = $1 and module = $2`, record.ID, module) return err } + +// givenOn is every machine port a module was given on this node by its `ports` setting, and which +// module it was given to. +func (i *Inventory) givenOn(ctx context.Context, nodeID any) (map[int]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select module, values->'ports' from settings + where node = $1 and jsonb_typeof(values->'ports') = 'object'`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int]string{} + for rows.Next() { + var module string + var raw []byte + if err := rows.Scan(&module, &raw); err != nil { + return nil, err + } + var given map[string]any + if err := json.Unmarshal(raw, &given); err != nil { + return nil, err + } + for _, v := range given { + if at, ok := v.(float64); ok { + out[int(at)] = module + } + } + } + return out, rows.Err() +} diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index ef2c8fa..b62f310 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -236,3 +236,24 @@ func TestUnassigningReleasesTheModulesPorts(t *testing.T) { t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err) } } + +// novox/hq ADR 0100: a port a node was given for a module is the node's, and the mesh never hands +// it to another. +func TestAGivenPortIsNeverAssigned(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + if err := inv.SetSettings(ctx, node, "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 20000}}); err != nil { + t.Fatal(err) + } + got, err := inv.PortFor(ctx, node, "web", 8080, false) + if err != nil { + t.Fatal(err) + } + if got.Machine == 20000 { + t.Fatal("a port given to postgres was assigned to web") + } + if _, err := inv.PortFor(ctx, node, "web", 20000, true); !errors.Is(err, ErrPortTaken) { + t.Fatalf("a fixed port given to another module was handed over: %v", err) + } +}