An address is read from the node's settings where it is used, never recorded with a port

Three readers did not follow a moved foundation port (novox/hq 04-ISSUES/102),
and each took the control-node down in its own way: the control plane's own
store and broker connections, sealed at genesis with the port inside; and every
build the mesh ever recorded, kept as `<registry>:<port>/<module>/<artifact>@…`.

The control plane cannot open its own sealed connections to move a port, and it
cannot bind the store as a consumer would — a binding mints a credential. So its
settings get a third twin: `NAME_PORT`, composed into its container from the
node's settings by a placeholder that names a seat, `${seat:mesh-store:5432}`,
and read on top of the sealed value by the store, the broker, the management API
and the bus connection. The answer is empty when the mesh has nothing to add,
so what genesis wrote stands until the node says otherwise.

A build is now recorded by digest and path — `artifact-store://<module>/<artifact>@…`
— and the store's address is composed in where a reference is used: the
declaration, the trust file, the bases a build is handed, a replay to the
catalogue. A reference recorded before this, with an address, is re-routed the
same way when the mesh built it. The trust file and every provider's address
now come from one derivation, with the node's given port over the mesh's
assignment over the manifest's number.

novox/hq 04-ISSUES/102
This commit is contained in:
2026-09-23 23:26:43 +02:00
parent 8fb32d7ee0
commit b7da02e370
20 changed files with 1463 additions and 46 deletions
+212
View File
@@ -0,0 +1,212 @@
package main
import (
"encoding/json"
"os"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// An address is read from the node's settings where it is used, never recorded with a port
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
var aDigest = "sha256:" + strings.Repeat("e", 64)
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
manifest, _ := json.Marshal(map[string]any{
"module": "gitea", "version": "1",
"resources": []map[string]any{
{"id": "server", "type": "container", "name": "mesh-gitea",
"image": "anchor.internal:5100/gitea/server@" + aDigest},
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
"image": "valkey/valkey@" + aDigest},
},
})
kept := buildFrom(link.BuildResult{
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
Manifest: manifest,
Made: []link.MadeArtifact{
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
},
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
})
if kept.Module != "gitea" {
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
}
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
}
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
}
recorded, err := catalogue.ParseManifest(kept.Manifest)
if err != nil {
t.Fatal(err)
}
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
t.Errorf("the recorded manifest's image is %v", got)
}
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
t.Errorf("the recorded manifest's archive is %v", got)
}
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
t.Errorf("an image the build did not make was rewritten: %v", got)
}
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
}
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
}
}
// aStore is a module offering the artifact store on 5000, published the long way as the
// distribution module does, so a node may be given another number for it.
func aStore() catalogue.Manifest {
return catalogue.Manifest{Module: "distribution", Version: "1",
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
}
// **The trust a machine writes for the store, and the address every built image is fetched
// through, say the port the node gave the store** — not the catalogue's number.
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aStore())
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
t.Fatal(err)
}
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
t.Fatal(err)
}
// A module the mesh built, recorded by digest and path, running on the other machine.
if err := open.inventory.RecordBuild(ctx, inventory.Build{
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
Made: []inventory.Artifact{{Name: "server", Kind: "image",
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
}); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
t.Fatal(err)
}
on := map[string]bool{"anchor": true, "laptop": true}
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
if err != nil || !found || node != "anchor" || port != "5101" {
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
}
var trust string
for _, r := range composed(t, open, "laptop").Resources {
if r["path"] == "/etc/docker/daemon.json" {
trust, _ = r["content"].(string)
}
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
t.Errorf("the image the mesh built is fetched as %v", r["image"])
}
}
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
}
// And a replay to the catalogue says where the store is now.
announced, err := following{open}.Announceable(ctx)
if err != nil {
t.Fatal(err)
}
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
t.Fatalf("the replay announces %+v", announced)
}
}
// **The control plane's own connections say the port the node gave the store and the broker.**
//
// Composed from the control plane's own manifest against a real inventory: the store's module is
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
// container is told so beside the sealed connection genesis wrote.
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
control, err := m.Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
Reference: "registry.example/control@" + aDigest}})
if err != nil {
t.Fatal(err)
}
register(t, open, control)
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
{Port: 5672, From: catalogue.FromMesh}},
Guards: []int{15672},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
// The store's module is registered and given its port, and NOT assigned: the state genesis
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
t.Fatal(err)
}
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
t.Fatal(err)
}
var env map[string]any
for _, r := range composed(t, open, "anchor").Resources {
if r["id"] == "mesh-controller.server" {
env, _ = r["env"].(map[string]any)
}
}
if env == nil {
t.Fatal("the control plane's container is not in its own node's declaration")
}
for key, want := range map[string]string{
"MESH_STORE_INVENTORY_PORT": "6852",
"MESH_STORE_IDENTITY_PORT": "6852",
"MESH_STORE_LICENCES_PORT": "6852",
"MESH_BROKER_AMQP_PORT": "5679",
"MESH_BROKER_ADDRESS_PORT": "5671",
"MESH_BROKER_MANAGEMENT_PORT": "15672",
} {
if env[key] != want {
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
}
}
}
+38 -9
View File
@@ -68,6 +68,11 @@ func buildCommand(ctx context.Context, args []string) error {
}
// buildFrom turns what a builder said into what the mesh keeps.
//
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
// reference and composes the store's address back in where a reference is used. `against` is kept
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
func buildFrom(result link.BuildResult) inventory.Build {
kept := inventory.Build{
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
@@ -77,16 +82,21 @@ func buildFrom(result link.BuildResult) inventory.Build {
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path, Manifest: result.Manifest, Against: result.Against,
Path: result.Path, Against: result.Against,
}
var announced []inventory.Artifact
for _, made := range result.Made {
kept.Made = append(kept.Made, inventory.Artifact{
announced = append(announced, inventory.Artifact{
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
})
kept.Made = append(kept.Made, inventory.Artifact{
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
})
}
kept.Manifest = recordedManifest(result.Manifest, announced)
// The module name comes from the manifest, which only exists when the build got that far.
if len(result.Manifest) > 0 {
if m, err := catalogue.ParseManifest(result.Manifest); err == nil {
if len(kept.Manifest) > 0 {
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
kept.Module = m.Module
}
}
@@ -378,7 +388,8 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
}
defer open.Close()
inv := open.inventory
if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
kept := buildFrom(result)
if err := inv.RecordBuild(ctx, kept); err != nil {
return err
}
@@ -388,13 +399,15 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
}
for _, made := range result.Made {
// Said as recorded: what each artifact is, not where this builder happened to push it.
for _, made := range kept.Made {
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
}
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
// second thing to disagree about what a manifest is.
manifest, err := catalogue.ParseManifest(result.Manifest)
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
// holds references by digest and path and every declaration composes the store's address in.
manifest, err := catalogue.ParseManifest(kept.Manifest)
if err != nil {
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
@@ -481,6 +494,9 @@ type answers struct {
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
// than a build command refusing to start because a query did not run. So the store not opening is
// reported and the build goes ahead without it.
//
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
// base is recorded by digest and path, and a build machine needs something it can pull.
func heldBy(ctx context.Context) map[string]string {
open, err := openStores(ctx)
if err != nil {
@@ -494,5 +510,18 @@ func heldBy(ctx context.Context) map[string]string {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
return nil
}
return held
address, err := whereTheStoreIs(ctx, open.inventory)
if err != nil {
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
return held
}
if address == "" {
return held
}
routed := make(map[string]string, len(held))
for repository, reference := range held {
routed[repository] = catalogue.Rerouted(reference, address)
}
return routed
}
+44 -11
View File
@@ -440,8 +440,11 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
return out, nil
}
// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
// module providing it is assigned to a machine that is on the private network.
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
//
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
@@ -454,29 +457,59 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
if err != nil {
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
}
providers := map[string]string{} // module -> served port
providers := map[string]catalogue.Manifest{}
for name, m := range shelf {
served, offers := m.Serves[catalogue.ArtifactStoreProvision]
if !offers {
continue
}
if p, ok := served["port"]; ok {
providers[name] = fmt.Sprintf("%v", p)
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
providers[name] = m
}
}
if len(providers) == 0 {
return "", "", false, nil
}
// In a stated order, so two machines offering it would always answer the same one.
machines := make([]string, 0, len(on))
for machine := range on {
machines = append(machines, machine)
}
sort.Strings(machines)
for _, machine := range machines {
assigned, err := inv.Assigned(ctx, machine)
if err != nil {
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
}
for _, a := range assigned {
if p, ok := providers[a]; ok {
return machine, p, true, nil
m, offers := providers[a]
if !offers {
continue
}
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
if err != nil {
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
}
if p, ok := serves["port"]; ok {
return machine, fmt.Sprintf("%v", p), true, nil
}
}
}
return "", "", false, nil
}
// artifactStoreAddress is the artifact store as this network reaches it, `<node>.internal:<port>`,
// or "" when the mesh has none on its network yet — the address composed into every reference
// the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (string, error) {
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return "", err
}
on := map[string]bool{}
for name := range onNetwork {
on[name] = true
}
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
if err != nil || !found {
return "", err
}
return overlay.InternalName(node) + ":" + port, nil
}
+138 -20
View File
@@ -229,28 +229,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
// What that module says a consumer needs to know, with that node's settings on
// it: a port somebody moved on the provider is a port its consumers must be told
// about, and the two coming from different places is how they come to disagree.
assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
if err != nil {
return catalogue.World{}, err
}
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
if err != nil {
return catalogue.World{}, err
}
// A port that node was given is where its consumers reach it (novox/hq ADR
// 0100). Unreadable given ports are that node's refusal to report, not this one's.
if given, err := catalogue.GivenPorts(m, layers); err == nil {
for wanted, at := range given {
assigned[wanted] = at
}
}
serves := catalogue.ServedOn(m, name, assigned)
if len(serves) > 0 {
serves, err = catalogue.Settle(serves, layers)
if err != nil {
return catalogue.World{}, err
}
}
offered[name] = append(offered[name], catalogue.Provider{
Node: o.node.Name, At: o.node.At, Serves: serves})
}
@@ -499,6 +481,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
return catalogue.Rendering{}, inventory.Node{}, err
}
// Where this node put the foundation's servers, for the control plane's own connections
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// And the artifact store as this network reaches it now — the address every image and
// archive the mesh built is fetched through, composed here and recorded nowhere — with what
// the mesh has built, so a reference recorded with an address before that is re-routed too.
artifactStore, err := artifactStoreAddress(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
held, err := inv.Held(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
built := make(map[string]bool, len(held))
for repository := range held {
built[repository] = true
}
// And every machine's name, so a container can reach one. The same set that writes the
// machine's own hosts file — one reading, so a container and its machine cannot disagree
// about where another machine is.
@@ -570,7 +575,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
Given: given, Taken: taken,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
}, record, nil
}
@@ -967,6 +972,119 @@ func managerPublicKeyFor(
return inv.SealingKeyOf(ctx, node)
}
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
// assigned over the manifest's own, settled with the node's settings layers.
//
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
// given ports are that node's refusal to report, not this reader's.
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
m catalogue.Manifest, provision string) (map[string]any, error) {
ports, layers, err := portsGivenOn(ctx, inv, node, m)
if err != nil {
return nil, err
}
serves := catalogue.ServedOn(m, provision, ports)
if len(serves) > 0 {
serves, err = catalogue.Settle(serves, layers)
if err != nil {
return nil, err
}
}
return serves, nil
}
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
// node's settings layers for the module, read once for both.
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
ports, err := portsOn(ctx, inv, node, m.Module)
if err != nil {
return nil, nil, err
}
layers, err := inv.SettingsFor(ctx, node, m.Module)
if err != nil {
return nil, nil, err
}
if given, err := catalogue.GivenPorts(m, layers); err == nil {
for wanted, at := range given {
ports[wanted] = at
}
}
return ports, layers, nil
}
// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the
// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102).
//
// Read for every module in the catalogue that claims a seat, in this node's set or not: the store
// and the broker are given their ports at genesis, as settings on a module that may be registered
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
node string, inSet []catalogue.Manifest) (map[string]map[int]int, error) {
assigned := map[string]bool{}
for _, m := range inSet {
assigned[m.Module] = true
}
names := make([]string, 0, len(shelf))
for name := range shelf {
names = append(names, name)
}
sort.Strings(names)
seats := map[string]map[int]int{}
for _, name := range names {
m := shelf[name]
var claims []string
for _, c := range m.Claims {
if c.At() == catalogue.ScopeMesh {
claims = append(claims, c.Name)
}
}
if len(claims) == 0 {
continue
}
ports, _, err := portsGivenOn(ctx, inv, node, m)
if err != nil {
return nil, err
}
if assigned[name] {
// Running here, so what its manifest publishes the long way is where this machine
// has it — the same reading the declaration itself makes (ADR 0038). Only for a
// holder in this node's set: a manifest's number says nothing about a machine the
// module does not run on.
var declared []int
for _, l := range m.Listens {
declared = append(declared, l.Port)
}
for _, wanted := range append(declared, m.Guards...) {
if _, said := ports[wanted]; said {
continue
}
if at, mayAssign := m.MachineSide(wanted); !mayAssign && at != 0 {
ports[wanted] = at
}
}
}
if len(ports) == 0 {
continue
}
for _, seat := range claims {
if seats[seat] == nil {
seats[seat] = map[int]int{}
}
for wanted, at := range ports {
if _, said := seats[seat][wanted]; said && !assigned[name] {
continue
}
seats[seat][wanted] = at
}
}
}
return seats, nil
}
// portsOn is one module's assignments on one machine, by the port the software uses.
func portsOn(
ctx context.Context, inv *inventory.Inventory, node, module string,
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"context"
"encoding/json"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// What a build is recorded as, and what it is announced and handed on as.
//
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
// — and the manifest with those references in it. The mesh records the digest and the path
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
// rebuild of everything the mesh has ever built.
// recordedManifest is a build's manifest with the store's address taken off every reference the
// build itself made. Other references — an image a module runs from a public registry — are what
// they were, which is why this rewrites only what `made` names rather than everything that looks
// like an address.
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
announced := map[string]bool{}
for _, a := range made {
announced[a.Reference] = true
}
return withReferences(raw, func(ref string) (string, bool) {
if !announced[ref] {
return ref, false
}
return catalogue.Recorded(ref), true
})
}
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
// composed into every reference the build made — recorded either way, before or after references
// were kept without their address.
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
recorded := map[string]bool{}
for _, a := range made {
recorded[catalogue.Recorded(a.Reference)] = true
}
return withReferences(raw, func(ref string) (string, bool) {
if !recorded[catalogue.Recorded(ref)] {
return ref, false
}
return catalogue.Rerouted(ref, address), true
})
}
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
// is, is the parser's to say, and it says so with a better sentence than anything here would.
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
if len(raw) == 0 {
return raw
}
var manifest map[string]any
if err := json.Unmarshal(raw, &manifest); err != nil {
return raw
}
resources, _ := manifest["resources"].([]any)
changed := false
for _, r := range resources {
resource, ok := r.(map[string]any)
if !ok {
continue
}
for _, key := range []string{"image", "source"} {
if written, ok := resource[key].(string); ok {
if rewritten, did := rewrite(written); did && rewritten != written {
resource[key] = rewritten
changed = true
}
}
}
}
if !changed {
return raw
}
out, err := json.Marshal(manifest)
if err != nil {
return raw
}
return out
}
// routedArtifacts is a build's artifacts as something can fetch them now.
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
if address == "" {
return made
}
out := make([]inventory.Artifact, 0, len(made))
for _, a := range made {
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
Reference: catalogue.Rerouted(a.Reference, address)})
}
return out
}
// whereTheStoreIs is the artifact store's address as this network reaches it, or "" — read for a
// caller that has the inventory open and nothing else in hand.
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory) (string, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return "", err
}
return artifactStoreAddress(ctx, inv, shelf)
}
+14 -1
View File
@@ -173,11 +173,21 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
// store the catalogue runs on, and the catalogue itself.
//
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
// the store's address, so a replay composes the address back in — the store's address as the
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
// store on the network yet, the recorded form goes as it is.
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
builds, err := f.open.inventory.Announceable(ctx)
if err != nil {
return nil, err
}
address, err := whereTheStoreIs(ctx, f.open.inventory)
if err != nil {
return nil, err
}
out := make([]link.Announcement, 0, len(builds))
for _, b := range builds {
a := link.Announcement{
@@ -186,8 +196,11 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
}
if len(b.Manifest) > 0 {
a.Manifest = b.Manifest
if address != "" {
a.Manifest = routedManifest(b.Manifest, b.Made, address)
}
}
for _, made := range b.Made {
for _, made := range routedArtifacts(b.Made, address) {
a.Made = append(a.Made, link.MadeArtifact{
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
})
+5 -1
View File
@@ -40,8 +40,12 @@ type Broker struct {
var ErrNotConfigured = errors.New("this control plane has not been told about its broker")
// FromEnvironment reads the two settings, if they are there.
//
// The address's port follows MESH_BROKER_ADDRESS_PORT when the node's settings moved the bus
// (novox/hq 04-ISSUES/102): the address genesis wrote is a public name and the port genesis
// chose, and only the port is the node's to move.
func FromEnvironment() (Broker, error) {
address, err := envfile.Value(AddressVar)
address, err := envfile.Placed(AddressVar)
if err != nil {
return Broker{}, err
}
+29
View File
@@ -178,3 +178,32 @@ func TestBothTogetherGiveABroker(t *testing.T) {
t.Errorf("got %+v", known)
}
}
// The node moved the bus, and the address a token carries follows (novox/hq 04-ISSUES/102).
func TestTheAddressPortFollowsThePortTwin(t *testing.T) {
t.Setenv(AddressVar, "broker.example:5671")
t.Setenv(AddressVar+"_FILE", "")
path, _ := writeCertificate(t)
t.Setenv(CertificateVar, path)
t.Setenv(AddressVar+"_PORT", "5679")
b, err := FromEnvironment()
if err != nil {
t.Fatal(err)
}
if b.Address != "broker.example:5679" {
t.Fatalf("the address is %q; the node put the bus on 5679", b.Address)
}
}
func TestTheManagementPortFollowsThePortTwin(t *testing.T) {
t.Setenv(ManagementVar, "http://guest:guest@127.0.0.1:15672")
t.Setenv(ManagementVar+"_FILE", "")
t.Setenv(ManagementVar+"_PORT", "15673")
m, err := ManagementFromEnvironment()
if err != nil {
t.Fatal(err)
}
if m.base.Host != "127.0.0.1:15673" {
t.Fatalf("the management API is at %q; the node put it on 15673", m.base.Host)
}
}
+4 -1
View File
@@ -41,8 +41,11 @@ type Management struct {
}
// ManagementFromEnvironment reads where the management API is, if it is configured.
//
// On the port MESH_BROKER_MANAGEMENT_PORT names when the node moved it (novox/hq 04-ISSUES/102);
// the URL's own port otherwise.
func ManagementFromEnvironment() (*Management, error) {
raw, err := envfile.Value(ManagementVar)
raw, err := envfile.Placed(ManagementVar)
if err != nil {
return nil, err
}
+136
View File
@@ -0,0 +1,136 @@
package catalogue
import (
"fmt"
"strings"
)
// What the mesh built, named by what it is rather than by where it was pushed.
//
// **An image is recorded by its digest and its path; the registry's address is a route to it**
// (novox/hq 04-ISSUES/102). A build used to be recorded as `<registry>:<port>/<module>/<artifact>@sha256:…`
// — the reference the builder pushed to, kept whole — and every declaration carried that literal.
// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new
// node, a recreate after eviction. The digest is the identity; the address is the node's setting
// for the module that serves the artifact store, and it is read from there when a reference is
// composed, never written into a record.
//
// So a kept reference has a scheme of the mesh's own, named after the provision that answers it:
//
// artifact-store://<module>/<artifact>@sha256:<hex> an image
// artifact-store://<module>/<artifact>/blobs/sha256:<hex> an archive
//
// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a
// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather
// than pulled from a public registry that happens to have a repository by that name — which is
// what an address-less `<module>/<artifact>@sha256:…` would be.
// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without
// the store's address.
const ArtifactStoreScheme = ArtifactStoreProvision + "://"
// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote
// one, taken off.
//
// For a reference the builder announced — one it pushed to the store — which is the only kind
// this is called on. An image the builder named `<host>/<path>@sha256:…` is kept as its path; an
// archive it named `http://<host>/v2/<path>/blobs/<digest>` likewise. A reference with no address
// in it — an image id from a genesis build that had nowhere to publish, a package version — is
// what it was.
func Recorded(reference string) string {
if strings.HasPrefix(reference, ArtifactStoreScheme) {
return reference
}
if rest, isURL := strings.CutPrefix(reference, "http://"); isURL {
if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") {
return ArtifactStoreScheme + path
}
return reference
}
host, path, ok := strings.Cut(reference, "/")
if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") {
return reference
}
return ArtifactStoreScheme + path
}
// isRegistryHost is the runtime's own rule for reading the first component of a reference as a
// registry rather than as a namespace: it has a dot or a port in it, or it is localhost.
func isRegistryHost(component string) bool {
return component == "localhost" || strings.ContainsAny(component, ".:")
}
// InArtifactStore reports whether a reference is a kept one, and what it names there.
func InArtifactStore(reference string) (path string, kept bool) {
return strings.CutPrefix(reference, ArtifactStoreScheme)
}
// Routed is a kept reference as a machine fetches it, through the artifact store at `address`
// (host:port). A reference that is not a kept one is what it was.
func Routed(reference, address string) string {
path, kept := InArtifactStore(reference)
if !kept {
return reference
}
if strings.Contains(path, "/blobs/") {
return "http://" + address + "/v2/" + path
}
return address + "/" + path
}
// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches
// it now: a kept one composed with the store's address, and one recorded before references were
// kept without their address — the builder's own `<host>/<path>@sha256:…` — re-routed to where
// the store is now. Only for references that are the mesh's own: everything a build record
// holds is, by construction.
func Rerouted(reference, address string) string {
return Routed(Recorded(reference), address)
}
// artifactsInto composes the artifact store's address into a resource's `image` and `source`.
//
// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is
// routed through the store as this network reaches it now. A reference recorded with an address
// before references were kept without one is re-routed the same way — but only when the mesh
// built it (`with.Built` names every `<module>/<artifact>` it has), because a module may run an
// image from a public registry under its own name and that one is exactly where it says.
//
// A kept reference with no store to route it through is refused: sent as it is, the runtime would
// refuse the scheme on the machine, one push away from the reason.
func artifactsInto(resource map[string]any, module string, with Rendering) error {
for _, key := range []string{"image", "source"} {
written, ok := resource[key].(string)
if !ok {
continue
}
if _, kept := InArtifactStore(written); kept {
if with.ArtifactStore == "" {
return fmt.Errorf(
"%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+
"artifact store on its network to fetch it from — nothing assigned offers "+
"%s, or the machine offering it is not on the private network",
module, resource["id"], written, ArtifactStoreProvision)
}
resource[key] = Routed(written, with.ArtifactStore)
continue
}
if with.ArtifactStore == "" {
continue
}
recorded := Recorded(written)
if recorded == written {
continue
}
path, _ := InArtifactStore(recorded)
repository := path
if at := strings.IndexAny(path, "@"); at >= 0 {
repository = path[:at]
} else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 {
repository = path[:blobs]
}
if with.Built[repository] {
resource[key] = Routed(recorded, with.ArtifactStore)
}
}
return nil
}
+137
View File
@@ -0,0 +1,137 @@
package catalogue
import (
"strings"
"testing"
)
// A build is recorded by what it is; where it is pushed is composed where it is used (novox/hq
// 04-ISSUES/102).
var digest = "sha256:" + strings.Repeat("d", 64)
func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) {
cases := map[string]string{
"anchor.internal:5100/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
"localhost:5000/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
"http://anchor.internal:5100/v2/gitea/config/blobs/" + digest: ArtifactStoreScheme + "gitea/config/blobs/" + digest,
ArtifactStoreScheme + "gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
digest: digest,
"@novox/sdk@1.2.3": "@novox/sdk@1.2.3",
"gitea/gitea@" + digest: "gitea/gitea@" + digest,
"https://registry.example/v2/gitea/config/blobs/" + digest: "https://registry.example/v2/gitea/config/blobs/" + digest,
}
for announced, want := range cases {
if got := Recorded(announced); got != want {
t.Errorf("Recorded(%q) = %q, want %q", announced, got, want)
}
}
}
func TestARecordedReferenceIsRoutedThroughTheStoreAsItIsNow(t *testing.T) {
if got := Routed(ArtifactStoreScheme+"gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
t.Errorf("an image is fetched as %q", got)
}
if got := Routed(ArtifactStoreScheme+"gitea/config/blobs/"+digest, "anchor.internal:5101"); got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
t.Errorf("an archive is fetched as %q", got)
}
if got := Routed("gitea/gitea@"+digest, "anchor.internal:5101"); got != "gitea/gitea@"+digest {
t.Errorf("a reference that is not the store's was routed: %q", got)
}
// One recorded before references were kept without their address follows the store too.
if got := Rerouted("anchor.internal:5100/gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
t.Errorf("a reference recorded with the old address stays there: %q", got)
}
}
// **The address is composed into a declaration, and the record never carries it.**
func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) {
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
{"id": "server", "type": "container", "name": "mesh-gitea", "artifact": "server"},
{"id": "config", "type": "archive", "path": "/etc/gitea", "artifact": "config"},
{"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + digest},
}, Build: &Build{Artifacts: []Artifact{
{Name: "server", Kind: ArtifactImage, From: "Dockerfile"},
{Name: "config", Kind: ArtifactArchive, From: "config"},
}}}
resolved, err := m.Resolve([]Built{
{Name: "server", Kind: ArtifactImage, Reference: ArtifactStoreScheme + "gitea/server@" + digest},
{Name: "config", Kind: ArtifactArchive, Reference: ArtifactStoreScheme + "gitea/config/blobs/" + digest, Digest: digest},
})
if err != nil {
t.Fatal(err)
}
r := Resolution{Node: "anchor", Modules: []Manifest{resolved}}
out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
if err != nil {
t.Fatal(err)
}
if got := fileNamed(out, "gitea.server")["image"]; got != "anchor.internal:5101/gitea/server@"+digest {
t.Errorf("the image the mesh built is fetched as %v", got)
}
if got := fileNamed(out, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
t.Errorf("the archive the mesh built is fetched from %v", got)
}
if got := fileNamed(out, "gitea.cache")["image"]; got != "valkey/valkey@"+digest {
t.Errorf("an image from a public registry was routed through the store: %v", got)
}
// The manifest the mesh holds still says what it is, not where it was fetched from.
if got := resolved.Resources[0]["image"]; got != ArtifactStoreScheme+"gitea/server@"+digest {
t.Errorf("composing wrote the address into the catalogue's copy: %v", got)
}
// And the store moves: the same record, another address, without a rebuild.
out, err = r.Declaration(Rendering{ArtifactStore: "laptop.internal:5000"})
if err != nil {
t.Fatal(err)
}
if got := fileNamed(out, "gitea.server")["image"]; got != "laptop.internal:5000/gitea/server@"+digest {
t.Errorf("after the store moved, the image is still fetched as %v", got)
}
}
func TestAnImageInTheStoreWithNoStoreToFetchItFromIsRefused(t *testing.T) {
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
{"id": "server", "type": "container", "name": "mesh-gitea",
"image": ArtifactStoreScheme + "gitea/server@" + digest},
}}
_, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(Rendering{})
if err == nil || !strings.Contains(err.Error(), "no artifact store") {
t.Fatalf("a reference nothing can fetch was sent to a machine: %v", err)
}
}
// **A reference recorded with an address before this follows the store too** — when the mesh
// built it. A module running an image straight from a public registry under its own name is left
// exactly where it says: `quay.io/keycloak/keycloak` is not the mesh's, whatever it is called.
func TestAReferenceRecordedWithAnAddressFollowsTheStoreWhenTheMeshBuiltIt(t *testing.T) {
m := Manifest{Module: "keycloak", Version: "1", Resources: []map[string]any{
{"id": "server", "type": "container", "name": "mesh-keycloak",
"image": "anchor.internal:5100/keycloak/server@" + digest},
{"id": "upstream", "type": "container", "name": "mesh-keycloak-upstream",
"image": "quay.io/keycloak/keycloak@" + digest},
}}
r := Resolution{Node: "anchor", Modules: []Manifest{m}}
out, err := r.Declaration(Rendering{
ArtifactStore: "anchor.internal:5101",
Built: map[string]bool{"keycloak/server": true},
})
if err != nil {
t.Fatal(err)
}
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5101/keycloak/server@"+digest {
t.Errorf("an image the mesh built, recorded with the old address, is fetched as %v", got)
}
if got := fileNamed(out, "keycloak.upstream")["image"]; got != "quay.io/keycloak/keycloak@"+digest {
t.Errorf("a public image was re-routed through the store: %v", got)
}
// Nothing known to be built: nothing re-routed, nothing refused.
out, err = r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
if err != nil {
t.Fatal(err)
}
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5100/keycloak/server@"+digest {
t.Errorf("with no build record, a reference was rewritten: %v", got)
}
}
+27
View File
@@ -143,6 +143,23 @@ type Rendering struct {
// from these only (ADR 0103): a port of a module assigned but not taken may still be the
// predecessor's.
Taken map[string]bool
// Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the
// holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and
// assignments for whichever module claims the seat, whether or not it is in this node's set.
// What ${seat:…} answers with; see seat_into.go for why the answer may be absent.
Seats map[string]map[int]int
// ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the
// node holding it and the port that node put it on — or empty when the mesh has none on its
// network yet. Composed into every image and archive the mesh built, at this moment and never
// stored (novox/hq 04-ISSUES/102).
ArtifactStore string
// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
// with an address — before references were kept without one — from an image a module runs
// straight from a public registry.
Built map[string]bool
}
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
@@ -539,6 +556,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
if err := portInto(copied, m.Module, m.Listens, with); err != nil {
return nil, err
}
// And where this machine put the foundation's servers, for the one module that
// reaches them by seat rather than by binding (novox/hq 04-ISSUES/102).
if err := seatInto(copied, m.Module, with); err != nil {
return nil, err
}
if err := machineInto(copied, thisMachine, m.Module); err != nil {
return nil, err
}
@@ -550,6 +572,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
if err := built(copied, m.Module); err != nil {
return nil, err
}
// What the mesh built is kept by digest and path; the store's address is this
// network's now, composed here and never recorded (novox/hq 04-ISSUES/102).
if err := artifactsInto(copied, m.Module, with); err != nil {
return nil, err
}
publishedOn(copied, m.Module, with)
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
// A service saying what it reflects names resources within its own module, so those
+120
View File
@@ -0,0 +1,120 @@
package catalogue
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
// Telling a module where this machine put the holder of a seat.
//
// **The foundation's ports are the node's** (novox/hq ADR 0100): the port a foundation server was
// given at genesis becomes that node's setting for the module that serves it, and every reader
// follows the setting. Every consumer's binding did. The control plane's own connections did not
// (04-ISSUES/102): they are written at genesis, before any module exists to bind to — full
// connection strings, sealed, with the port inside — so when the node moved the store, the
// control plane went on dialling where genesis had written and the mesh was headless.
//
// The control plane cannot open its own sealed connection to move the port, and it cannot bind
// the store as a consumer would: a binding mints a credential, and what the control plane holds
// is the foundation's superuser, made before the mesh. What it can do is read the node's settings
// when it composes its own declaration — it is the thing that composes every other module's — and
// say in its own environment which port this machine put the store at.
//
// So a module may ask about a **seat** (ADR 0079: a foundation seat is named after the server it
// guards — `mesh-store`, `mesh-broker`). `${seat:mesh-store:5432}` is "the port this machine put
// the holder of the mesh-store seat's 5432 at". Not a provision: nothing is required, nothing is
// granted, no credential is minted. A seat is the mesh's own vocabulary for the store and the
// broker, which is what makes this the control plane's way of naming them and not a way for a
// module to reach a server it was not granted — the answer is a port number the mesh holds in the
// clear, and the credential to use it is still the module's own to have.
//
// **The answer may be empty, and that is the one place a placeholder answers with nothing.** The
// store and the broker are raised at genesis, before the mesh knows them as modules; a mesh raised
// on the catalogue's own ports never gives them a setting at all. In both, the port genesis wrote
// into the connection string is the right one, and the mesh has nothing to add. An empty answer
// says exactly that, and what reads it — the control plane's `_PORT` twin — treats an empty value
// as no value. Answering with the software's own port instead would override what genesis wrote
// with a number the mesh never checked, on the one machine where that is a headless mesh.
// ofSeat is where a module asks about a seat: ${seat:<seat>:<the port its holder's software uses>}.
var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
// holder — in a file's content, and in a value of a container's environment. The same two places
// portInto fills, for the same reason: they are where a process reads a number from.
func seatInto(resource map[string]any, module string, with Rendering) error {
switch fmt.Sprint(resource["type"]) {
case "file":
content, ok := resource["content"].(string)
if !ok || !ofSeat.MatchString(content) {
return nil
}
filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with)
if err != nil {
return err
}
resource["content"] = filled
case "container":
env, ok := resource["env"].(map[string]any)
if !ok {
return nil
}
named := make([]string, 0, len(env))
for key := range env {
named = append(named, key)
}
sort.Strings(named)
// A fresh map, and only when something changes — this map is the catalogue's, shared by
// every node running the module (see portInto).
var filled map[string]any
for _, key := range named {
written, ok := env[key].(string)
if !ok || !ofSeat.MatchString(written) {
continue
}
value, err := seatsFilledInto(written,
fmt.Sprintf("%s's container %s sets %s to something that",
module, resource["name"], key), with)
if err != nil {
return err
}
if filled == nil {
filled = map[string]any{}
for k, v := range env {
filled[k] = v
}
}
filled[key] = value
}
if filled != nil {
resource["env"] = filled
}
}
return nil
}
// seatsFilledInto answers every ${seat:…} in one written value.
//
// A port the seat's holder does not publish on this machine — or a seat nothing on it holds —
// answers with nothing, for the reason the package comment gives. A port that is not one is
// refused: it was written by a person and it is wrong.
func seatsFilledInto(written, where string, with Rendering) (string, error) {
for _, m := range ofSeat.FindAllStringSubmatch(written, -1) {
seat, port := m[1], m[2]
wanted, err := strconv.Atoi(port)
if err != nil || wanted < 1 || wanted > 65535 {
return "", fmt.Errorf("%s says ${seat:%s:%s}, and %s is not a port", where, seat, port, port)
}
answer := ""
if at, known := with.Seats[seat][wanted]; known {
answer = strconv.Itoa(at)
}
written = strings.ReplaceAll(written, m[0], answer)
}
return written, nil
}
+157
View File
@@ -0,0 +1,157 @@
package catalogue
import (
"os"
"strings"
"testing"
)
// The control plane's own addresses follow the node's ports (novox/hq 04-ISSUES/102).
func TestASeatPlaceholderAnswersWhereThisMachinePutTheHolder(t *testing.T) {
control := map[string]any{
"type": "container", "id": "server", "name": "mesh-controller",
"env": map[string]any{
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
"MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
},
}
with := Rendering{Seats: map[string]map[int]int{
"mesh-store": {5432: 6852}, "mesh-broker": {5672: 5679, 5671: 5671},
}}
if err := seatInto(control, "mesh-controller", with); err != nil {
t.Fatal(err)
}
env := control["env"].(map[string]any)
for key, want := range map[string]string{
"MESH_STORE_INVENTORY_PORT": "6852",
"MESH_BROKER_AMQP_PORT": "5679",
"MESH_BROKER_ADDRESS_PORT": "5671",
"MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
} {
if env[key] != want {
t.Errorf("%s = %v, want %q", key, env[key], want)
}
}
}
// A seat nothing on this machine holds — or one whose holder the mesh has given no port — answers
// with nothing, so the port genesis wrote into the connection string stands. Not the software's
// own port: on a node given a port at genesis before the store's module exists, that would
// override the right number with the catalogue's.
func TestASeatTheMeshCannotPlaceAnswersWithNothing(t *testing.T) {
control := map[string]any{
"type": "container", "id": "server", "name": "mesh-controller",
"env": map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"},
}
if err := seatInto(control, "mesh-controller", Rendering{}); err != nil {
t.Fatal(err)
}
if got := control["env"].(map[string]any)["MESH_STORE_INVENTORY_PORT"]; got != "" {
t.Fatalf("with nothing known, the seat answered %q", got)
}
file := map[string]any{"type": "file", "content": "port=${seat:mesh-store:5432}\n"}
if err := seatInto(file, "x", Rendering{Seats: map[string]map[int]int{"mesh-store": {5433: 1}}}); err != nil {
t.Fatal(err)
}
if got := file["content"]; got != "port=\n" {
t.Fatalf("a port the holder does not publish answered %q", got)
}
}
func TestASeatPlaceholderNamingNoPortIsRefused(t *testing.T) {
file := map[string]any{"type": "file", "content": "${seat:mesh-store:99999}"}
if err := seatInto(file, "x", Rendering{}); err == nil {
t.Fatal("99999 was accepted as a port")
}
}
func TestFillingASeatLeavesTheManifestAlone(t *testing.T) {
env := map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"}
manifest := map[string]any{"type": "container", "id": "server", "env": env}
for _, at := range []int{6852, 5432} {
copied := map[string]any{}
for k, v := range manifest {
copied[k] = v
}
with := Rendering{Seats: map[string]map[int]int{"mesh-store": {5432: at}}}
if err := seatInto(copied, "mesh-controller", with); err != nil {
t.Fatal(err)
}
}
if env["MESH_STORE_INVENTORY_PORT"] != "${seat:mesh-store:5432}" {
t.Fatalf("the module's own manifest was edited: %v", env)
}
}
// **The control plane's own manifest, composed through the whole path.**
//
// The store was given 6852 and the broker's plain port 5679 (the control-node's migration, novox/hq
// 04-ISSUES/102). The control plane's own connections are sealed at genesis with the ports genesis
// wrote; what its container is told beside them is where this machine put the store and the
// broker now, read from the node's settings exactly as every consumer's binding is.
func TestTheControlPlanesOwnAddressesFollowTheNodesPorts(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Fatalf("the control plane's own manifest does not parse:\n%v", err)
}
control, err := m.Resolve([]Built{{
Name: "server", Kind: ArtifactImage,
Reference: ArtifactStoreScheme + "mesh-controller/server@sha256:" + strings.Repeat("c", 64),
}})
if err != nil {
t.Fatal(err)
}
r := Resolution{Node: "anchor", Modules: []Manifest{control}}
needed := map[string]map[string]string{"mesh-controller": {}}
for name := range m.OwnSecrets {
needed["mesh-controller"][name] = "sealed-" + name
}
out, err := r.Declaration(Rendering{
Needed: needed,
ArtifactStore: "anchor.internal:5100",
Seats: map[string]map[int]int{
"mesh-store": {5432: 6852},
"mesh-broker": {5671: 5671, 5672: 5679, 15672: 15673},
},
})
if err != nil {
t.Fatalf("the control plane does not compose: %v", err)
}
server := fileNamed(out, "mesh-controller.server")
if server == nil {
t.Fatalf("the control plane's container is not in the declaration: %v", out)
}
env, _ := server["env"].(map[string]any)
for key, want := range map[string]string{
"MESH_STORE_INVENTORY_PORT": "6852",
"MESH_STORE_IDENTITY_PORT": "6852",
"MESH_STORE_LICENCES_PORT": "6852",
"MESH_BROKER_AMQP_PORT": "5679",
"MESH_BROKER_MANAGEMENT_PORT": "15673",
"MESH_BROKER_ADDRESS_PORT": "5671",
} {
if env[key] != want {
t.Errorf("the control plane is told %s=%v; the node put it on %s", key, env[key], want)
}
}
if got := server["image"]; got != "anchor.internal:5100/mesh-controller/server@sha256:"+strings.Repeat("c", 64) {
t.Errorf("the control plane's own image is %v, not routed through the store", got)
}
// And on a mesh where the foundation is where genesis raised it, nothing is added.
out, err = r.Declaration(Rendering{Needed: needed, ArtifactStore: "anchor.internal:5100"})
if err != nil {
t.Fatal(err)
}
env, _ = fileNamed(out, "mesh-controller.server")["env"].(map[string]any)
if env["MESH_STORE_INVENTORY_PORT"] != "" || env["MESH_BROKER_AMQP_PORT"] != "" {
t.Errorf("with no settings, the control plane is told %v", env)
}
}
+123
View File
@@ -0,0 +1,123 @@
package envfile
import (
"fmt"
"net"
"os"
"regexp"
"strconv"
"strings"
)
// The port a machine put something on, said beside the value that names it.
//
// **An address written down when a port was decided does not follow the port** (novox/hq
// 04-ISSUES/102). The control plane's own store and broker connections are written at genesis —
// full connection strings, password and all — and sealed, so the mesh cannot open them to move the
// port inside. When the node's settings move that port, every consumer's binding follows and the
// control plane's own connection does not: it goes on dialling the number genesis wrote, and the
// mesh is headless.
//
// So a setting has a third twin. `NAME_FILE` says where the value is; `NAME_PORT` says which port
// this machine put the thing at, composed into the control plane's environment from the node's
// settings exactly as a consumer's binding is. The value's own port is what stands when the twin
// says nothing — a mesh whose foundation is still where genesis raised it needs no override, and
// an empty answer is the mesh saying it has nothing to add, not the mesh saying zero.
//
// Only the port. The host inside the value is the machine's own loopback, or the broker's public
// name — a fact of the genesis, not of any node's settings — and the mesh has no better opinion
// of it. What moves under ADR 0100 is the port.
// PortVar is the twin saying which port this machine put the named thing at.
func PortVar(name string) string { return name + "_PORT" }
// Port is what NAME_PORT says, checked to be a port, or "" when it says nothing.
//
// Blank counts as unset, as it does for every other variable here: a container given an empty
// string was given nothing, and refusing it as ambiguous would turn an omission into a puzzle.
func Port(name string) (string, error) {
raw := strings.TrimSpace(os.Getenv(PortVar(name)))
if raw == "" {
return "", nil
}
n, err := strconv.Atoi(raw)
if err != nil || n < 1 || n > 65535 {
return "", fmt.Errorf("%s is %q, which is not a port", PortVar(name), raw)
}
return strconv.Itoa(n), nil
}
// Placed is Value, with its port moved to where NAME_PORT says this machine put it.
func Placed(name string) (string, error) {
value, err := Value(name)
if err != nil {
return "", err
}
port, err := Port(name)
if err != nil || port == "" {
return value, err
}
placed, err := WithPort(value, port)
if err != nil {
// Where it came from is said; what it says is not. The value is a connection string with
// a password in it, and every error here is written on the assumption it will be logged.
return "", fmt.Errorf("%s names a port to move %s to, and %s could not be read as "+
"something with a port in it: %w", PortVar(name), name, name, err)
}
return placed, nil
}
// keywordPort is `port=…` in a `key=value` connection string.
var keywordPort = regexp.MustCompile(`(^|\s)port=\S*`)
// WithPort is the value with its port replaced by `port`.
//
// Three shapes, because the control plane's settings come in three: a URL
// (`scheme://[user:password@]host[:port][/…]`), a bare `host[:port]` (the broker's address) and
// a `key=value` connection string (`host=… port=…`), which is what the store's driver accepts
// beside a URL. An IPv6 host stays in its brackets. Nothing here parses the URL properly — a
// password with a character a URL parser dislikes is still a working connection string, and
// refusing it would refuse a value that has been dialling fine since genesis.
func WithPort(value, port string) (string, error) {
value = strings.TrimSpace(value)
if value == "" {
return "", fmt.Errorf("the value is empty")
}
if scheme, rest, isURL := strings.Cut(value, "://"); isURL {
end := strings.IndexAny(rest, "/?#")
authority, tail := rest, ""
if end >= 0 {
authority, tail = rest[:end], rest[end:]
}
userinfo := ""
if at := strings.LastIndex(authority, "@"); at >= 0 {
userinfo, authority = authority[:at+1], authority[at+1:]
}
host, err := hostWithPort(authority, port)
if err != nil {
return "", err
}
return scheme + "://" + userinfo + host + tail, nil
}
if strings.Contains(value, "=") && !strings.ContainsAny(value, "/") {
if keywordPort.MatchString(value) {
return keywordPort.ReplaceAllString(value, "${1}port="+port), nil
}
return value + " port=" + port, nil
}
return hostWithPort(value, port)
}
// hostWithPort is `host[:port]` with the port set, brackets kept around an IPv6 host.
func hostWithPort(authority, port string) (string, error) {
if authority == "" {
return "", fmt.Errorf("there is no host to put a port on")
}
host, _, err := net.SplitHostPort(authority)
if err != nil {
// No port yet. A bracketed IPv6 host without a port is a shape SplitHostPort refuses, and
// a bare one carries colons of its own — both are a host, not an error.
host = strings.TrimSuffix(strings.TrimPrefix(authority, "["), "]")
}
return net.JoinHostPort(host, port), nil
}
+66
View File
@@ -0,0 +1,66 @@
package envfile
import (
"os"
"path/filepath"
"strings"
"testing"
)
// The control plane's own connections follow the port the node moved (novox/hq 04-ISSUES/102).
func TestAPortTwinMovesTheValuesPort(t *testing.T) {
cases := map[string]string{
"postgres://mesh:s3cret@127.0.0.1:5432/inventory?sslmode=disable": "postgres://mesh:s3cret@127.0.0.1:6852/inventory?sslmode=disable",
"postgres://mesh:s3cret@127.0.0.1/inventory": "postgres://mesh:s3cret@127.0.0.1:6852/inventory",
"amqp://control:p%40ss@127.0.0.1:5672/": "amqp://control:p%40ss@127.0.0.1:6852/",
"amqp://control:p@ss:with@127.0.0.1:5672/": "amqp://control:p@ss:with@127.0.0.1:6852/",
"http://guest:guest@127.0.0.1:15672": "http://guest:guest@127.0.0.1:6852",
"http://[::1]:15672/api": "http://[::1]:6852/api",
"broker.example:5671": "broker.example:6852",
"broker.example": "broker.example:6852",
"host=127.0.0.1 port=5432 dbname=inventory": "host=127.0.0.1 port=6852 dbname=inventory",
"host=127.0.0.1 dbname=inventory": "host=127.0.0.1 dbname=inventory port=6852",
}
for value, want := range cases {
got, err := WithPort(value, "6852")
if err != nil || got != want {
t.Errorf("WithPort(%q) = %q, %v; want %q", value, got, err, want)
}
}
}
func TestPlacedLeavesTheValueAloneWhenNothingSaysAPort(t *testing.T) {
path := filepath.Join(t.TempDir(), "value")
if err := os.WriteFile(path, []byte("postgres://m:p@127.0.0.1:5432/inventory\n"), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("MESH_P_FILE", path)
t.Setenv("MESH_P_PORT", "")
got, err := Placed("MESH_P")
if err != nil || got != "postgres://m:p@127.0.0.1:5432/inventory" {
t.Fatalf("got %q, %v", got, err)
}
t.Setenv("MESH_P_PORT", " 6852 ")
got, err = Placed("MESH_P")
if err != nil || got != "postgres://m:p@127.0.0.1:6852/inventory" {
t.Fatalf("got %q, %v", got, err)
}
}
func TestAPortTwinThatIsNotAPortIsRefusedWithoutQuotingTheValue(t *testing.T) {
t.Setenv("MESH_Q", "postgres://m:hunter2@127.0.0.1:5432/inventory")
t.Setenv("MESH_Q_PORT", "many")
_, err := Placed("MESH_Q")
if err == nil {
t.Fatal("a port that is not a number was accepted")
}
if strings.Contains(err.Error(), "hunter2") {
t.Fatalf("the value was quoted back: %v", err)
}
t.Setenv("MESH_Q", "")
t.Setenv("MESH_Q_PORT", "6852")
if _, err := Placed("MESH_Q"); err == nil {
t.Fatal("a port with no value to put it on was accepted")
}
}
+5 -1
View File
@@ -146,8 +146,12 @@ func (s *Server) Answers(r Replayer) error {
}
// Connect opens the control plane's own connection to the broker.
//
// On the port MESH_BROKER_AMQP_PORT names when the node's settings moved the broker (novox/hq
// 04-ISSUES/102) — the URL is genesis's, sealed, and its port is the one thing in it the node may
// have moved since.
func Connect(enroller Enroller, listener Listener) (*Server, error) {
url, err := envfile.Value(AMQPVar)
url, err := envfile.Placed(AMQPVar)
if err != nil {
return nil, err
}
+52
View File
@@ -213,3 +213,55 @@ func mustNotLeak(t *testing.T, err error) {
t.Fatalf("the password is in the error: %v", err)
}
}
// The node moved the store, and the control plane's own connection follows (novox/hq 04-ISSUES/102).
//
// The connection string is what genesis wrote, port and all; the port twin is what the node's
// settings say now. The pool's configuration is the one place both meet.
func TestThePortTwinMovesTheStoresPort(t *testing.T) {
alone(t)
t.Setenv(PortVariable(example), "")
path := filepath.Join(t.TempDir(), "inventory")
if err := os.WriteFile(path, []byte(dsn+"\n"), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv(FileVariable(example), path)
opened, err := Open(t.Context(), example)
if err != nil {
t.Fatal(err)
}
if got := opened.Pool().Config().ConnConfig.Port; got != 5432 {
t.Fatalf("with nothing said, the store is on %d rather than what the file says", got)
}
opened.Close()
t.Setenv(PortVariable(example), "6852")
opened, err = Open(t.Context(), example)
if err != nil {
t.Fatalf("a moved port was refused: %v", err)
}
defer opened.Close()
if got := opened.Pool().Config().ConnConfig.Port; got != 6852 {
t.Fatalf("the store is on %d, and the node put it on 6852", got)
}
if got := opened.Pool().Config().ConnConfig.Password; got != "s3cret-in-here" {
t.Fatalf("moving the port changed the password to %q", got)
}
}
func TestAPortTwinThatIsNotAPortNamesItselfAndNotTheSecret(t *testing.T) {
alone(t)
t.Setenv(Variable(example), dsn)
t.Setenv(PortVariable(example), "six")
_, err := Open(t.Context(), example)
if err == nil {
t.Fatal("a port that is not a number was accepted")
}
if !strings.Contains(err.Error(), PortVariable(example)) {
t.Errorf("the error does not name the variable: %v", err)
}
if strings.Contains(err.Error(), "s3cret") {
t.Errorf("the error quotes the password: %v", err)
}
}
+37 -1
View File
@@ -25,6 +25,8 @@ import (
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/novox/mesh-controller/internal/envfile"
)
// contextName is what a context may be called.
@@ -67,6 +69,17 @@ func Variable(context string) string {
// raises the first one.
func FileVariable(context string) string { return Variable(context) + "_FILE" }
// PortVariable is the environment variable naming the PORT this machine put the store at — the
// third twin, beside the value and the file.
//
// **The connection string is sealed and the port inside it is genesis's** (novox/hq 04-ISSUES/102).
// The control plane cannot open its own store secret to move the port, and a node's settings can
// move the store (ADR 0100: the foundation's ports are the node's). Every consumer's binding
// followed that setting; the control plane's own connection did not, and the mesh was headless. So
// the port is composed into the control plane's environment from the node's settings, exactly as a
// consumer's binding is, and the connection string's own port stands only when this says nothing.
func PortVariable(context string) string { return envfile.PortVar(Variable(context)) }
// Database is what a context's database is called.
//
// Named after the context, so that a person looking at a PostgreSQL server can see which
@@ -85,7 +98,7 @@ func Open(ctx context.Context, name string) (*Store, error) {
"name, so it must be lower-case letters and digits, starting with a letter", name)
}
dsn, from, err := settingsFor(name)
dsn, from, err := placed(name)
if err != nil {
return nil, err
}
@@ -169,6 +182,29 @@ func settingsFor(name string) (dsn, from string, err error) {
return direct, Variable(name), nil
}
// placed is a context's connection string with its port moved to where this machine put the
// store, when the node's settings say so (PortVariable), and as it was otherwise.
func placed(name string) (dsn, from string, err error) {
dsn, from, err = settingsFor(name)
if err != nil {
return "", "", err
}
port, err := envfile.Port(Variable(name))
if err != nil || port == "" {
return dsn, from, err
}
moved, err := envfile.WithPort(dsn, port)
if err != nil {
// The variable and the port are named; the connection string is not, for the same reason
// as everywhere else in this file.
return "", "", fmt.Errorf(
"%s says this machine put the %s store on port %s, and the connection settings in %s "+
"could not be read as something with a port in them: %w",
PortVariable(name), name, port, from, err)
}
return moved, from + ", on port " + port + " as " + PortVariable(name) + " says", nil
}
// Context is which context this store belongs to.
func (s *Store) Context() string { return s.context }
+7 -1
View File
@@ -42,7 +42,13 @@
"MESH_STORE_LICENCES_FILE": "/run/secrets/licences",
"MESH_BROKER_AMQP_FILE": "/run/secrets/broker",
"MESH_BROKER_MANAGEMENT_FILE": "/run/secrets/broker-management",
"MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address"
"MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address",
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}"
},
"volumes": [
"mesh-broker-tls:/broker-tls:ro",