diff --git a/cmd/mesh-controller/addresses_test.go b/cmd/mesh-controller/addresses_test.go new file mode 100644 index 0000000..fbe76d2 --- /dev/null +++ b/cmd/mesh-controller/addresses_test.go @@ -0,0 +1,335 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/link" +) + +// An address is read from the node's settings where it is used, never recorded with a port +// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here. + +var aDigest = "sha256:" + strings.Repeat("e", 64) + +// **A build is recorded by digest and path**, whatever address the builder pushed to — and only +// what the build made is rewritten: an image the module runs from elsewhere is left where it says. +func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) { + manifest, _ := json.Marshal(map[string]any{ + "module": "gitea", "version": "1", + "resources": []map[string]any{ + {"id": "server", "type": "container", "name": "mesh-gitea", + "image": "anchor.internal:5100/gitea/server@" + aDigest}, + {"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest, + "source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest}, + {"id": "cache", "type": "container", "name": "mesh-gitea-cache", + "image": "valkey/valkey@" + aDigest}, + }, + }) + kept := buildFrom(link.BuildResult{ + ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop", + Manifest: manifest, + Made: []link.MadeArtifact{ + {Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest}, + {Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest}, + }, + Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest}, + }) + if kept.Module != "gitea" { + t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module) + } + if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest { + t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference) + } + if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest { + t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference) + } + recorded, err := catalogue.ParseManifest(kept.Manifest) + if err != nil { + t.Fatal(err) + } + if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest { + t.Errorf("the recorded manifest's image is %v", got) + } + if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest { + t.Errorf("the recorded manifest's archive is %v", got) + } + if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest { + t.Errorf("an image the build did not make was rewritten: %v", got) + } + if strings.Contains(string(kept.Manifest), "anchor.internal:5100") { + t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest) + } + if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest { + t.Errorf("what the build stood on was rewritten: %v", kept.Against) + } +} + +// aStore is a module offering the artifact store on 5000, published the long way as the +// distribution module does, so a node may be given another number for it. +func aStore() catalogue.Manifest { + return catalogue.Manifest{Module: "distribution", Version: "1", + Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}}, + Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}}, + Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry", + "ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}} +} + +// **The trust a machine writes for the store, and the address every built image is fetched +// through, say the port the node gave the store** — not the catalogue's number. +func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aStore()) + if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { + t.Fatal(err) + } + // A module the mesh built, recorded by digest and path, running on the other machine. + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "app", Commit: "abc", + Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "app", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", + "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + + on := map[string]bool{"anchor": true, "laptop": true} + node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) + if err != nil || !found || node != "anchor" || port != "5101" { + t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) + } + + var trust string + for _, r := range composed(t, open, "laptop").Resources { + if r["path"] == "/etc/docker/daemon.json" { + trust, _ = r["content"].(string) + } + if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { + t.Errorf("the image the mesh built is fetched as %v", r["image"]) + } + } + if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { + t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) + } + + // And a replay to the catalogue says where the store is now. + announced, err := following{open}.Announceable(ctx) + if err != nil { + t.Fatal(err) + } + if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { + t.Fatalf("the replay announces %+v", announced) + } +} + +// **The control plane's own connections say the port the node gave the store and the broker.** +// +// Composed from the control plane's own manifest against a real inventory: the store's module is +// given 6852 on this node the way genesis or an operator gives it, and the control plane's +// container is told so beside the sealed connection genesis wrote. +func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage, + Reference: "registry.example/control@" + aDigest}}) + if err != nil { + t.Fatal(err) + } + register(t, open, control) + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) + register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, + {Port: 5672, From: catalogue.FromMesh}}, + Guards: []int{15672}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", + "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) + if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + // The store's module is registered and given its port, and NOT assigned: the state genesis + // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). + if err := open.inventory.SetSettings(ctx, "anchor", "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", + map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { + t.Fatal(err) + } + + var env map[string]any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "mesh-controller.server" { + env, _ = r["env"].(map[string]any) + } + } + if env == nil { + t.Fatal("the control plane's container is not in its own node's declaration") + } + for key, want := range map[string]string{ + "MESH_STORE_INVENTORY_PORT": "6852", + "MESH_STORE_IDENTITY_PORT": "6852", + "MESH_STORE_LICENCES_PORT": "6852", + "MESH_BROKER_AMQP_PORT": "5679", + // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, + // because the sealed value beside it carries the port genesis wrote (finding F3). + "MESH_BROKER_ADDRESS_PORT": "", + "MESH_BROKER_MANAGEMENT_PORT": "", + } { + if env[key] != want { + t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) + } + } +} + +// withSeatPorts is the control plane's manifest with the seat placeholders in its environment — +// added here until module.json carries them (the manifest lands one commit after the code that +// fills it, so a control plane one build behind never sees a placeholder it cannot fill). +func withSeatPorts(m catalogue.Manifest) catalogue.Manifest { + seatPorts := map[string]string{ + "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}", + "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", + "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", + "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", + } + out := m + out.Resources = nil + for _, r := range m.Resources { + if r["type"] != "container" { + out.Resources = append(out.Resources, r) + continue + } + copied := map[string]any{} + for k, v := range r { + copied[k] = v + } + env := map[string]any{} + if had, ok := r["env"].(map[string]any); ok { + for k, v := range had { + env[k] = v + } + } + for k, v := range seatPorts { + if _, said := env[k]; !said { + env[k] = v + } + } + copied["env"] = env + out.Resources = append(out.Resources, copied) + } + return out +} + +// aLoneNode is one capable machine with nothing placed on any network — the control-node during +// genesis, before the "network" step, which is after the store, the broker, the vault and the +// catalogue have each been built and pushed (finding F2). +func aLoneNode(t *testing.T) *stores { + t.Helper() + inventory.ForTest(t) + licences.ForTest(t) + open, err := openStores(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(open.Close) + for _, m := range provided { + if err := open.inventory.Provide(t.Context(), m); err != nil { + t.Fatal(err) + } + } + record, err := open.inventory.AddNode(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ + {"name": "container-runtime", "present": true}}}) + var profile map[string]any + _ = json.Unmarshal(reported, &profile) + if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + return open +} + +// **Before the network exists, the store's own node reaches it by loopback** — never refused, +// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to +// a node on no network, and builds the catalogue on a base it must be able to pull. +func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { + open := aLoneNode(t) + ctx := t.Context() + register(t, open, aStore()) + register(t, open, catalogue.Manifest{Module: "builder", Version: "1", + Requires: []string{catalogue.ArtifactStoreProvision}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", + "image": "registry.example/mesh-builder@" + aDigest}}}) + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", + Made: []inventory.Artifact{{Name: "runtime", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", + "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) + for _, module := range []string{"distribution", "builder", "postgres"} { + if _, err := assign(ctx, open, "anchor", module); err != nil { + t.Fatal(err) + } + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { + t.Fatal(err) + } + + var image any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "postgres.runtime" { + image = r["image"] + } + } + if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) + } + held := heldBy(ctx) + if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("a builder beside the store is handed the base %q", got) + } +} diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index a612778..98dbf38 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -68,6 +68,11 @@ func buildCommand(ctx context.Context, args []string) error { } // buildFrom turns what a builder said into what the mesh keeps. +// +// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder +// says `://@sha256:…`; the mesh records the artifact-store +// reference and composes the store's address back in where a reference is used. `against` is kept +// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge. func buildFrom(result link.BuildResult) inventory.Build { kept := inventory.Build{ ID: result.ID, Repository: result.Repository, Ref: result.Ref, @@ -77,16 +82,21 @@ func buildFrom(result link.BuildResult) inventory.Build { // edges, and it is not always listening when a build happens — on a fresh mesh it cannot // be, for exactly the modules it needs most. Keeping them is what makes a replay able to // rebuild the graph rather than a list of names. - Path: result.Path, Manifest: result.Manifest, Against: result.Against, + Path: result.Path, Against: result.Against, } + var announced []inventory.Artifact for _, made := range result.Made { - kept.Made = append(kept.Made, inventory.Artifact{ + announced = append(announced, inventory.Artifact{ Name: made.Name, Kind: made.Kind, Reference: made.Reference, }) + kept.Made = append(kept.Made, inventory.Artifact{ + Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference), + }) } + kept.Manifest = recordedManifest(result.Manifest, announced) // The module name comes from the manifest, which only exists when the build got that far. - if len(result.Manifest) > 0 { - if m, err := catalogue.ParseManifest(result.Manifest); err == nil { + if len(kept.Manifest) > 0 { + if m, err := catalogue.ParseManifest(kept.Manifest); err == nil { kept.Module = m.Module } } @@ -378,7 +388,8 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat } defer open.Close() inv := open.inventory - if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil { + kept := buildFrom(result) + if err := inv.RecordBuild(ctx, kept); err != nil { return err } @@ -388,13 +399,15 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) } - for _, made := range result.Made { + // Said as recorded: what each artifact is, not where this builder happened to push it. + for _, made := range kept.Made { fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference) } // Parsed with the same parser a hand-written manifest goes through. A second path would be a - // second thing to disagree about what a manifest is. - manifest, err := catalogue.ParseManifest(result.Manifest) + // second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue + // holds references by digest and path and every declaration composes the store's address in. + manifest, err := catalogue.ParseManifest(kept.Manifest) if err != nil { return fmt.Errorf("%s built %s and what came back is not a manifest: %w", result.On, result.Repository, err) @@ -481,6 +494,9 @@ type answers struct { // all, and one that does gets a refusal naming exactly what is missing — which is a better sentence // than a build command refusing to start because a query did not run. So the store not opening is // reported and the build goes ahead without it. +// +// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a +// base is recorded by digest and path, and a build machine needs something it can pull. func heldBy(ctx context.Context) map[string]string { open, err := openStores(ctx) if err != nil { @@ -494,5 +510,18 @@ func heldBy(ctx context.Context) map[string]string { fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err) return nil } - return held + address, err := whereABuilderReachesTheStore(ctx, open.inventory) + if err != nil { + fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+ + "module naming a base will be handed a reference nothing can fetch: %v\n", err) + return held + } + if address == "" { + return held + } + routed := make(map[string]string, len(held)) + for repository, reference := range held { + routed[repository] = catalogue.Rerouted(reference, address) + } + return routed } diff --git a/cmd/mesh-controller/network.go b/cmd/mesh-controller/network.go index 7f62c24..681a124 100644 --- a/cmd/mesh-controller/network.go +++ b/cmd/mesh-controller/network.go @@ -440,8 +440,11 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory, return out, nil } -// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a -// module providing it is assigned to a machine that is on the private network. +// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the +// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100, +// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when +// neither says anything. Read exactly as a consumer's binding is, because the trust a machine +// writes for the store and the address it pulls from are the same fact as what a consumer is told. // // A lookup failure is an error, never "not found": collapsing the two composed a declaration // without the trust whenever the inventory hiccuped, delivered by a push that reported success — @@ -454,29 +457,87 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory, if err != nil { return "", "", false, fmt.Errorf("reading the catalogue: %w", err) } - providers := map[string]string{} // module -> served port + providers := map[string]catalogue.Manifest{} for name, m := range shelf { - served, offers := m.Serves[catalogue.ArtifactStoreProvision] - if !offers { - continue - } - if p, ok := served["port"]; ok { - providers[name] = fmt.Sprintf("%v", p) + if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers { + providers[name] = m } } if len(providers) == 0 { return "", "", false, nil } + // In a stated order, so two machines offering it would always answer the same one. + machines := make([]string, 0, len(on)) for machine := range on { + machines = append(machines, machine) + } + sort.Strings(machines) + for _, machine := range machines { assigned, err := inv.Assigned(ctx, machine) if err != nil { return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err) } for _, a := range assigned { - if p, ok := providers[a]; ok { - return machine, p, true, nil + m, offers := providers[a] + if !offers { + continue + } + serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision) + if err != nil { + return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err) + } + if p, ok := serves["port"]; ok { + return machine, fmt.Sprintf("%v", p), true, nil } } } return "", "", false, nil } + +// artifactStoreAddress is the artifact store as `forNode` reaches it: `.internal:` +// over the private network, or — when nothing is on the network yet — `127.0.0.1:` for the +// node that holds the store itself, and "" for any other. The address composed into every +// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102). +// +// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each +// of those is built and pushed to a node that is on no network, and the store is on that same +// node; an answer of "no store" there would refuse every one of those pushes and hand every one +// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the +// address genesis itself reaches the store by. +func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory, + shelf map[string]catalogue.Manifest, forNode string) (string, error) { + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return "", err + } + on := map[string]bool{} + for name := range onNetwork { + on[name] = true + } + node, port, found, err := artifactStoreOnNetwork(ctx, inv, on) + if err != nil { + return "", err + } + if found { + return overlay.InternalName(node) + ":" + port, nil + } + holder, port, found, err := artifactStoreHolder(ctx, inv) + if err != nil || !found || holder != forNode { + return "", err + } + return "127.0.0.1:" + port, nil +} + +// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or +// off the network, and the port that node put it on. +func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) { + nodes, err := inv.Nodes(ctx) + if err != nil { + return "", "", false, err + } + all := map[string]bool{} + for _, n := range nodes { + all[n.Name] = true + } + return artifactStoreOnNetwork(ctx, inv, all) +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 4b5cbbf..d066c27 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -229,28 +229,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, // What that module says a consumer needs to know, with that node's settings on // it: a port somebody moved on the provider is a port its consumers must be told // about, and the two coming from different places is how they come to disagree. - assigned, err := portsOn(ctx, inv, o.node.Name, m.Module) + serves, err := servedOnNode(ctx, inv, o.node.Name, m, name) if err != nil { return catalogue.World{}, err } - layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) - if err != nil { - return catalogue.World{}, err - } - // A port that node was given is where its consumers reach it (novox/hq ADR - // 0100). Unreadable given ports are that node's refusal to report, not this one's. - if given, err := catalogue.GivenPorts(m, layers); err == nil { - for wanted, at := range given { - assigned[wanted] = at - } - } - serves := catalogue.ServedOn(m, name, assigned) - if len(serves) > 0 { - serves, err = catalogue.Settle(serves, layers) - if err != nil { - return catalogue.World{}, err - } - } offered[name] = append(offered[name], catalogue.Provider{ Node: o.node.Name, At: o.node.At, Serves: serves}) } @@ -499,6 +481,22 @@ func renderingFor(ctx context.Context, open *stores, node string, return catalogue.Rendering{}, inventory.Node{}, err } + // The artifact store as this node reaches it now — the address every image and archive the + // mesh built is fetched through, composed here and recorded nowhere — with what the mesh has + // built, so a reference recorded with an address before that is re-routed too. + artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + held, err := inv.Held(ctx) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + built := make(map[string]bool, len(held)) + for repository := range held { + built[repository] = true + } + // And every machine's name, so a container can reach one. The same set that writes the // machine's own hosts file — one reading, so a container and its machine cannot disagree // about where another machine is. @@ -566,11 +564,18 @@ func renderingFor(ctx context.Context, open *stores, node string, taken[m] = true } } + // Where this node put the foundation's servers, for the control plane's own connections + // (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat, + // exactly as a consumer's binding is, never from what genesis wrote into a secret. + seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, - Given: given, Taken: taken, + Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, }, record, nil } @@ -967,6 +972,119 @@ func managerPublicKeyFor( return inv.SealingKeyOf(ctx, node) } +// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT +// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh +// assigned over the manifest's own, settled with the node's settings layers. +// +// **The one derivation** for every reader of a provider's address — a consumer's binding, the +// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable +// given ports are that node's refusal to report, not this reader's. +func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string, + m catalogue.Manifest, provision string) (map[string]any, error) { + ports, layers, err := portsGivenOn(ctx, inv, node, m) + if err != nil { + return nil, err + } + serves := catalogue.ServedOn(m, provision, ports) + if len(serves) > 0 { + serves, err = catalogue.Settle(serves, layers) + if err != nil { + return nil, err + } + } + return serves, nil +} + +// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the +// node's settings layers for the module, read once for both. +func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string, + m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) { + ports, err := portsOn(ctx, inv, node, m.Module) + if err != nil { + return nil, nil, err + } + layers, err := inv.SettingsFor(ctx, node, m.Module) + if err != nil { + return nil, nil, err + } + if given, err := catalogue.GivenPorts(m, layers); err == nil { + for wanted, at := range given { + ports[wanted] = at + } + } + return ports, layers, nil +} + +// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the +// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102). +// +// Read for every module in the catalogue that claims a seat, in this node's set or not: the store +// and the broker are given their ports at genesis, as settings on a module that may be registered +// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the +// first declaration it composes for itself. A holder in this node's set wins over one that is not. +func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, + node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) { + assigned := map[string]bool{} + for _, m := range inSet { + assigned[m.Module] = true + } + names := make([]string, 0, len(shelf)) + for name := range shelf { + names = append(names, name) + } + sort.Strings(names) + seats := map[string]map[int]int{} + for _, name := range names { + m := shelf[name] + var claims []string + for _, c := range m.Claims { + if c.At() == catalogue.ScopeMesh { + claims = append(claims, c.Name) + } + } + if len(claims) == 0 { + continue + } + // **Only a port the node was given or the mesh assigned — never the manifest's own + // number.** The sealed value the answer sits beside carries the port genesis wrote, which + // on a given-port node is the predecessor's; a manifest's long-form mapping is the + // catalogue's default, and answering with it would override the right number with one + // the mesh never checked (the contract in seat_into.go). And on an adopted node a holder + // assigned but not yet taken is the found container, on the ports it was found with, not + // the declaration's — so its mesh-assigned ports do not count there either; a given port + // does, because a given port is the found one by construction (ADR 0100). + ports, _, err := portsGivenOn(ctx, inv, node, m) + if err != nil { + return nil, err + } + if adopted && !taken[name] { + layers, err := inv.SettingsFor(ctx, node, m.Module) + if err != nil { + return nil, err + } + ports = map[int]int{} + if given, err := catalogue.GivenPorts(m, layers); err == nil { + ports = given + } + } + if len(ports) == 0 { + continue + } + for _, seat := range claims { + if seats[seat] == nil { + seats[seat] = map[int]int{} + } + for wanted, at := range ports { + if _, said := seats[seat][wanted]; said && !assigned[name] { + continue + } + seats[seat][wanted] = at + } + } + } + return seats, nil +} + // portsOn is one module's assignments on one machine, by the port the software uses. func portsOn( ctx context.Context, inv *inventory.Inventory, node, module string, diff --git a/cmd/mesh-controller/references.go b/cmd/mesh-controller/references.go new file mode 100644 index 0000000..202f2cf --- /dev/null +++ b/cmd/mesh-controller/references.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "encoding/json" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// What a build is recorded as, and what it is announced and handed on as. +// +// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder +// announces each artifact by the reference it pushed — `://@sha256:…` +// — and the manifest with those references in it. The mesh records the digest and the path +// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder +// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is +// given. Nothing recorded carries a port, so moving the store is a settings change and not a +// rebuild of everything the mesh has ever built. + +// recordedManifest is a build's manifest with the store's address taken off every reference the +// build itself made. Other references — an image a module runs from a public registry — are what +// they were, which is why this rewrites only what `made` names rather than everything that looks +// like an address. +func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage { + announced := map[string]bool{} + for _, a := range made { + announced[a.Reference] = true + } + return withReferences(raw, func(ref string) (string, bool) { + if !announced[ref] { + return ref, false + } + return catalogue.Recorded(ref), true + }) +} + +// routedManifest is a recorded manifest with the store's address, as this network reaches it now, +// composed into every reference the build made — recorded either way, before or after references +// were kept without their address. +func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage { + recorded := map[string]bool{} + for _, a := range made { + recorded[catalogue.Recorded(a.Reference)] = true + } + return withReferences(raw, func(ref string) (string, bool) { + if !recorded[catalogue.Recorded(ref)] { + return ref, false + } + return catalogue.Rerouted(ref, address), true + }) +} + +// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest +// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest +// is, is the parser's to say, and it says so with a better sentence than anything here would. +func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage { + if len(raw) == 0 { + return raw + } + var manifest map[string]any + if err := json.Unmarshal(raw, &manifest); err != nil { + return raw + } + resources, _ := manifest["resources"].([]any) + changed := false + for _, r := range resources { + resource, ok := r.(map[string]any) + if !ok { + continue + } + for _, key := range []string{"image", "source"} { + if written, ok := resource[key].(string); ok { + if rewritten, did := rewrite(written); did && rewritten != written { + resource[key] = rewritten + changed = true + } + } + } + } + if !changed { + return raw + } + out, err := json.Marshal(manifest) + if err != nil { + return raw + } + return out +} + +// routedArtifacts is a build's artifacts as something can fetch them now. +func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact { + if address == "" { + return made + } + out := make([]inventory.Artifact, 0, len(made)) + for _, a := range made { + out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind, + Reference: catalogue.Rerouted(a.Reference, address)}) + } + return out +} + +// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" — +// read for a caller that has the inventory open and nothing else in hand. With no node named, the +// store's own node: loopback when nothing is on the network yet. +func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return "", err + } + if forNode == "" { + if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil { + return "", err + } else if found { + forNode = holder + } + } + return artifactStoreAddress(ctx, inv, shelf, forNode) +} + +// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's +// when there is one, else loopback on the store's own node — when that node also holds a module +// requiring the store, which is what a builder is (genesis: one node holds both). +func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return "", err + } + holder, _, found, err := artifactStoreHolder(ctx, inv) + if err != nil || !found { + return "", err + } + assigned, err := inv.Assigned(ctx, holder) + if err != nil { + return "", err + } + besideIt := false + for _, a := range assigned { + for _, r := range shelf[a].Requires { + if r == catalogue.ArtifactStoreProvision { + besideIt = true + } + } + } + if !besideIt { + holder = "" + } + return artifactStoreAddress(ctx, inv, shelf, holder) +} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 3476864..d58d685 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -173,11 +173,21 @@ func sayUpgrade(module string, u inventory.Upgrade) string { // catalogue misses nothing — but the modules built before it first ran were announced to a queue // that did not exist, and on a fresh mesh those are always the same three: the shared base, the // store the catalogue runs on, and the catalogue itself. +// +// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is +// recorded by digest and path; the catalogue hears the builder's own announcements, which name +// the store's address, so a replay composes the address back in — the store's address as the +// network reaches it NOW, which is the whole point of not having recorded the old one. With no +// store on the network yet, the recorded form goes as it is. func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) { builds, err := f.open.inventory.Announceable(ctx) if err != nil { return nil, err } + address, err := whereTheStoreIs(ctx, f.open.inventory, "") + if err != nil { + return nil, err + } out := make([]link.Announcement, 0, len(builds)) for _, b := range builds { a := link.Announcement{ @@ -186,8 +196,11 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error } if len(b.Manifest) > 0 { a.Manifest = b.Manifest + if address != "" { + a.Manifest = routedManifest(b.Manifest, b.Made, address) + } } - for _, made := range b.Made { + for _, made := range routedArtifacts(b.Made, address) { a.Made = append(a.Made, link.MadeArtifact{ Name: made.Name, Kind: made.Kind, Reference: made.Reference, }) diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 51e5c87..93d5b54 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -40,8 +40,12 @@ type Broker struct { var ErrNotConfigured = errors.New("this control plane has not been told about its broker") // FromEnvironment reads the two settings, if they are there. +// +// The address's port follows MESH_BROKER_ADDRESS_PORT when the node's settings moved the bus +// (novox/hq 04-ISSUES/102): the address genesis wrote is a public name and the port genesis +// chose, and only the port is the node's to move. func FromEnvironment() (Broker, error) { - address, err := envfile.Value(AddressVar) + address, err := envfile.Placed(AddressVar) if err != nil { return Broker{}, err } diff --git a/internal/broker/broker_test.go b/internal/broker/broker_test.go index 94e8810..8baf100 100644 --- a/internal/broker/broker_test.go +++ b/internal/broker/broker_test.go @@ -178,3 +178,32 @@ func TestBothTogetherGiveABroker(t *testing.T) { t.Errorf("got %+v", known) } } + +// The node moved the bus, and the address a token carries follows (novox/hq 04-ISSUES/102). +func TestTheAddressPortFollowsThePortTwin(t *testing.T) { + t.Setenv(AddressVar, "broker.example:5671") + t.Setenv(AddressVar+"_FILE", "") + path, _ := writeCertificate(t) + t.Setenv(CertificateVar, path) + t.Setenv(AddressVar+"_PORT", "5679") + b, err := FromEnvironment() + if err != nil { + t.Fatal(err) + } + if b.Address != "broker.example:5679" { + t.Fatalf("the address is %q; the node put the bus on 5679", b.Address) + } +} + +func TestTheManagementPortFollowsThePortTwin(t *testing.T) { + t.Setenv(ManagementVar, "http://guest:guest@127.0.0.1:15672") + t.Setenv(ManagementVar+"_FILE", "") + t.Setenv(ManagementVar+"_PORT", "15673") + m, err := ManagementFromEnvironment() + if err != nil { + t.Fatal(err) + } + if m.base.Host != "127.0.0.1:15673" { + t.Fatalf("the management API is at %q; the node put it on 15673", m.base.Host) + } +} diff --git a/internal/broker/management.go b/internal/broker/management.go index cea2a44..2693b69 100644 --- a/internal/broker/management.go +++ b/internal/broker/management.go @@ -41,8 +41,11 @@ type Management struct { } // ManagementFromEnvironment reads where the management API is, if it is configured. +// +// On the port MESH_BROKER_MANAGEMENT_PORT names when the node moved it (novox/hq 04-ISSUES/102); +// the URL's own port otherwise. func ManagementFromEnvironment() (*Management, error) { - raw, err := envfile.Value(ManagementVar) + raw, err := envfile.Placed(ManagementVar) if err != nil { return nil, err } diff --git a/internal/catalogue/artifacts.go b/internal/catalogue/artifacts.go new file mode 100644 index 0000000..e9a010d --- /dev/null +++ b/internal/catalogue/artifacts.go @@ -0,0 +1,145 @@ +package catalogue + +import ( + "fmt" + "strings" +) + +// What the mesh built, named by what it is rather than by where it was pushed. +// +// **An image is recorded by its digest and its path; the registry's address is a route to it** +// (novox/hq 04-ISSUES/102). A build used to be recorded as `://@sha256:…` +// — the reference the builder pushed to, kept whole — and every declaration carried that literal. +// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new +// node, a recreate after eviction. The digest is the identity; the address is the node's setting +// for the module that serves the artifact store, and it is read from there when a reference is +// composed, never written into a record. +// +// So a kept reference has a scheme of the mesh's own, named after the provision that answers it: +// +// artifact-store:///@sha256: an image +// artifact-store:////blobs/sha256: an archive +// +// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a +// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather +// than pulled from a public registry that happens to have a repository by that name — which is +// what an address-less `/@sha256:…` would be. + +// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without +// the store's address. +const ArtifactStoreScheme = ArtifactStoreProvision + "://" + +// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote +// one, taken off. +// +// For a reference the builder announced — one it pushed to the store — which is the only kind +// this is called on. An image the builder named `/@sha256:…` is kept as its path; an +// archive it named `http:///v2//blobs/` likewise. A reference with no address +// in it — an image id from a genesis build that had nowhere to publish, a package version — is +// what it was. +func Recorded(reference string) string { + if strings.HasPrefix(reference, ArtifactStoreScheme) { + return reference + } + if rest, isURL := strings.CutPrefix(reference, "http://"); isURL { + if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") { + return ArtifactStoreScheme + path + } + return reference + } + host, path, ok := strings.Cut(reference, "/") + if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") { + return reference + } + return ArtifactStoreScheme + path +} + +// isRegistryHost is the runtime's own rule for reading the first component of a reference as a +// registry rather than as a namespace: it has a dot or a port in it, or it is localhost. +func isRegistryHost(component string) bool { + return component == "localhost" || strings.ContainsAny(component, ".:") +} + +// InArtifactStore reports whether a reference is a kept one, and what it names there. +func InArtifactStore(reference string) (path string, kept bool) { + return strings.CutPrefix(reference, ArtifactStoreScheme) +} + +// Routed is a kept reference as a machine fetches it, through the artifact store at `address` +// (host:port). A reference that is not a kept one is what it was. +func Routed(reference, address string) string { + path, kept := InArtifactStore(reference) + if !kept { + return reference + } + if strings.Contains(path, "/blobs/") { + return "http://" + address + "/v2/" + path + } + return address + "/" + path +} + +// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches +// it now: a kept one composed with the store's address, and one recorded before references were +// kept without their address — the builder's own `/@sha256:…` — re-routed to where +// the store is now. Only for references that are the mesh's own: everything a build record +// holds is, by construction. +func Rerouted(reference, address string) string { + return Routed(Recorded(reference), address) +} + +// artifactsInto composes the artifact store's address into a resource's `image` and `source`. +// +// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is +// routed through the store as this network reaches it now. A reference recorded with an address +// before references were kept without one is re-routed the same way — but only when the mesh +// built it (`with.Built` names every `/` it has), because a module may run an +// image from a public registry under its own name and that one is exactly where it says. +// +// A kept reference with no store to route it through is refused: sent as it is, the runtime would +// refuse the scheme on the machine, one push away from the reason. +// +// **What this does not reach: the images genesis pinned.** The installer builds the control plane +// and the builder before the mesh exists, pushes them itself and pins their manifests to +// `:/mesh-controller@…` and `:/mesh-builder@…` — single-segment +// repositories with no build record, so `with.Built` does not name them and they are left as +// written until each is rebuilt through the mesh, which records it by digest and path. Until then +// a registry that moves strands exactly those two on a recreate, and the control plane's is the +// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the +// store (novox/hq 04-ISSUES/102, finding F4). +func artifactsInto(resource map[string]any, module string, with Rendering) error { + for _, key := range []string{"image", "source"} { + written, ok := resource[key].(string) + if !ok { + continue + } + if _, kept := InArtifactStore(written); kept { + if with.ArtifactStore == "" { + return fmt.Errorf( + "%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+ + "artifact store on its network to fetch it from — nothing assigned offers "+ + "%s, or the machine offering it is not on the private network", + module, resource["id"], written, ArtifactStoreProvision) + } + resource[key] = Routed(written, with.ArtifactStore) + continue + } + if with.ArtifactStore == "" { + continue + } + recorded := Recorded(written) + if recorded == written { + continue + } + path, _ := InArtifactStore(recorded) + repository := path + if at := strings.IndexAny(path, "@"); at >= 0 { + repository = path[:at] + } else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 { + repository = path[:blobs] + } + if with.Built[repository] { + resource[key] = Routed(recorded, with.ArtifactStore) + } + } + return nil +} diff --git a/internal/catalogue/artifacts_test.go b/internal/catalogue/artifacts_test.go new file mode 100644 index 0000000..3d0ae70 --- /dev/null +++ b/internal/catalogue/artifacts_test.go @@ -0,0 +1,137 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A build is recorded by what it is; where it is pushed is composed where it is used (novox/hq +// 04-ISSUES/102). + +var digest = "sha256:" + strings.Repeat("d", 64) + +func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) { + cases := map[string]string{ + "anchor.internal:5100/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest, + "localhost:5000/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest, + "http://anchor.internal:5100/v2/gitea/config/blobs/" + digest: ArtifactStoreScheme + "gitea/config/blobs/" + digest, + ArtifactStoreScheme + "gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest, + digest: digest, + "@novox/sdk@1.2.3": "@novox/sdk@1.2.3", + "gitea/gitea@" + digest: "gitea/gitea@" + digest, + "https://registry.example/v2/gitea/config/blobs/" + digest: "https://registry.example/v2/gitea/config/blobs/" + digest, + } + for announced, want := range cases { + if got := Recorded(announced); got != want { + t.Errorf("Recorded(%q) = %q, want %q", announced, got, want) + } + } +} + +func TestARecordedReferenceIsRoutedThroughTheStoreAsItIsNow(t *testing.T) { + if got := Routed(ArtifactStoreScheme+"gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest { + t.Errorf("an image is fetched as %q", got) + } + if got := Routed(ArtifactStoreScheme+"gitea/config/blobs/"+digest, "anchor.internal:5101"); got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest { + t.Errorf("an archive is fetched as %q", got) + } + if got := Routed("gitea/gitea@"+digest, "anchor.internal:5101"); got != "gitea/gitea@"+digest { + t.Errorf("a reference that is not the store's was routed: %q", got) + } + // One recorded before references were kept without their address follows the store too. + if got := Rerouted("anchor.internal:5100/gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest { + t.Errorf("a reference recorded with the old address stays there: %q", got) + } +} + +// **The address is composed into a declaration, and the record never carries it.** +func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) { + m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{ + {"id": "server", "type": "container", "name": "mesh-gitea", "artifact": "server"}, + {"id": "config", "type": "archive", "path": "/etc/gitea", "artifact": "config"}, + {"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + digest}, + }, Build: &Build{Artifacts: []Artifact{ + {Name: "server", Kind: ArtifactImage, From: "Dockerfile"}, + {Name: "config", Kind: ArtifactArchive, From: "config"}, + }}} + resolved, err := m.Resolve([]Built{ + {Name: "server", Kind: ArtifactImage, Reference: ArtifactStoreScheme + "gitea/server@" + digest}, + {Name: "config", Kind: ArtifactArchive, Reference: ArtifactStoreScheme + "gitea/config/blobs/" + digest, Digest: digest}, + }) + if err != nil { + t.Fatal(err) + } + r := Resolution{Node: "anchor", Modules: []Manifest{resolved}} + + out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"}) + if err != nil { + t.Fatal(err) + } + if got := fileNamed(out, "gitea.server")["image"]; got != "anchor.internal:5101/gitea/server@"+digest { + t.Errorf("the image the mesh built is fetched as %v", got) + } + if got := fileNamed(out, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest { + t.Errorf("the archive the mesh built is fetched from %v", got) + } + if got := fileNamed(out, "gitea.cache")["image"]; got != "valkey/valkey@"+digest { + t.Errorf("an image from a public registry was routed through the store: %v", got) + } + // The manifest the mesh holds still says what it is, not where it was fetched from. + if got := resolved.Resources[0]["image"]; got != ArtifactStoreScheme+"gitea/server@"+digest { + t.Errorf("composing wrote the address into the catalogue's copy: %v", got) + } + + // And the store moves: the same record, another address, without a rebuild. + out, err = r.Declaration(Rendering{ArtifactStore: "laptop.internal:5000"}) + if err != nil { + t.Fatal(err) + } + if got := fileNamed(out, "gitea.server")["image"]; got != "laptop.internal:5000/gitea/server@"+digest { + t.Errorf("after the store moved, the image is still fetched as %v", got) + } +} + +func TestAnImageInTheStoreWithNoStoreToFetchItFromIsRefused(t *testing.T) { + m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{ + {"id": "server", "type": "container", "name": "mesh-gitea", + "image": ArtifactStoreScheme + "gitea/server@" + digest}, + }} + _, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(Rendering{}) + if err == nil || !strings.Contains(err.Error(), "no artifact store") { + t.Fatalf("a reference nothing can fetch was sent to a machine: %v", err) + } +} + +// **A reference recorded with an address before this follows the store too** — when the mesh +// built it. A module running an image straight from a public registry under its own name is left +// exactly where it says: `quay.io/keycloak/keycloak` is not the mesh's, whatever it is called. +func TestAReferenceRecordedWithAnAddressFollowsTheStoreWhenTheMeshBuiltIt(t *testing.T) { + m := Manifest{Module: "keycloak", Version: "1", Resources: []map[string]any{ + {"id": "server", "type": "container", "name": "mesh-keycloak", + "image": "anchor.internal:5100/keycloak/server@" + digest}, + {"id": "upstream", "type": "container", "name": "mesh-keycloak-upstream", + "image": "quay.io/keycloak/keycloak@" + digest}, + }} + r := Resolution{Node: "anchor", Modules: []Manifest{m}} + out, err := r.Declaration(Rendering{ + ArtifactStore: "anchor.internal:5101", + Built: map[string]bool{"keycloak/server": true}, + }) + if err != nil { + t.Fatal(err) + } + if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5101/keycloak/server@"+digest { + t.Errorf("an image the mesh built, recorded with the old address, is fetched as %v", got) + } + if got := fileNamed(out, "keycloak.upstream")["image"]; got != "quay.io/keycloak/keycloak@"+digest { + t.Errorf("a public image was re-routed through the store: %v", got) + } + // Nothing known to be built: nothing re-routed, nothing refused. + out, err = r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"}) + if err != nil { + t.Fatal(err) + } + if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5100/keycloak/server@"+digest { + t.Errorf("with no build record, a reference was rewritten: %v", got) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index a9b5549..42d6938 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -143,6 +143,23 @@ type Rendering struct { // from these only (ADR 0103): a port of a module assigned but not taken may still be the // predecessor's. Taken map[string]bool + + // Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the + // holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and + // assignments for whichever module claims the seat, whether or not it is in this node's set. + // What ${seat:…} answers with; see seat_into.go for why the answer may be absent. + Seats map[string]map[int]int + + // ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the + // node holding it and the port that node put it on — or empty when the mesh has none on its + // network yet. Composed into every image and archive the mesh built, at this moment and never + // stored (novox/hq 04-ISSUES/102). + ArtifactStore string + + // Built is every `/` the mesh has built. What tells a reference recorded + // with an address — before references were kept without one — from an image a module runs + // straight from a public registry. + Built map[string]bool } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has @@ -539,6 +556,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri if err := portInto(copied, m.Module, m.Listens, with); err != nil { return nil, err } + // And where this machine put the foundation's servers, for the one module that + // reaches them by seat rather than by binding (novox/hq 04-ISSUES/102). + if err := seatInto(copied, m.Module, with); err != nil { + return nil, err + } if err := machineInto(copied, thisMachine, m.Module); err != nil { return nil, err } @@ -550,6 +572,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri if err := built(copied, m.Module); err != nil { return nil, err } + // What the mesh built is kept by digest and path; the store's address is this + // network's now, composed here and never recorded (novox/hq 04-ISSUES/102). + if err := artifactsInto(copied, m.Module, with); err != nil { + return nil, err + } publishedOn(copied, m.Module, with) copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) // A service saying what it reflects names resources within its own module, so those diff --git a/internal/catalogue/seat_into.go b/internal/catalogue/seat_into.go new file mode 100644 index 0000000..c127c9d --- /dev/null +++ b/internal/catalogue/seat_into.go @@ -0,0 +1,120 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +// Telling a module where this machine put the holder of a seat. +// +// **The foundation's ports are the node's** (novox/hq ADR 0100): the port a foundation server was +// given at genesis becomes that node's setting for the module that serves it, and every reader +// follows the setting. Every consumer's binding did. The control plane's own connections did not +// (04-ISSUES/102): they are written at genesis, before any module exists to bind to — full +// connection strings, sealed, with the port inside — so when the node moved the store, the +// control plane went on dialling where genesis had written and the mesh was headless. +// +// The control plane cannot open its own sealed connection to move the port, and it cannot bind +// the store as a consumer would: a binding mints a credential, and what the control plane holds +// is the foundation's superuser, made before the mesh. What it can do is read the node's settings +// when it composes its own declaration — it is the thing that composes every other module's — and +// say in its own environment which port this machine put the store at. +// +// So a module may ask about a **seat** (ADR 0079: a foundation seat is named after the server it +// guards — `mesh-store`, `mesh-broker`). `${seat:mesh-store:5432}` is "the port this machine put +// the holder of the mesh-store seat's 5432 at". Not a provision: nothing is required, nothing is +// granted, no credential is minted. A seat is the mesh's own vocabulary for the store and the +// broker, which is what makes this the control plane's way of naming them and not a way for a +// module to reach a server it was not granted — the answer is a port number the mesh holds in the +// clear, and the credential to use it is still the module's own to have. +// +// **The answer may be empty, and that is the one place a placeholder answers with nothing.** The +// store and the broker are raised at genesis, before the mesh knows them as modules; a mesh raised +// on the catalogue's own ports never gives them a setting at all. In both, the port genesis wrote +// into the connection string is the right one, and the mesh has nothing to add. An empty answer +// says exactly that, and what reads it — the control plane's `_PORT` twin — treats an empty value +// as no value. Answering with the software's own port instead would override what genesis wrote +// with a number the mesh never checked, on the one machine where that is a headless mesh. + +// ofSeat is where a module asks about a seat: ${seat::}. +var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`) + +// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's +// holder — in a file's content, and in a value of a container's environment. The same two places +// portInto fills, for the same reason: they are where a process reads a number from. +func seatInto(resource map[string]any, module string, with Rendering) error { + switch fmt.Sprint(resource["type"]) { + case "file": + content, ok := resource["content"].(string) + if !ok || !ofSeat.MatchString(content) { + return nil + } + filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with) + if err != nil { + return err + } + resource["content"] = filled + + case "container": + env, ok := resource["env"].(map[string]any) + if !ok { + return nil + } + named := make([]string, 0, len(env)) + for key := range env { + named = append(named, key) + } + sort.Strings(named) + + // A fresh map, and only when something changes — this map is the catalogue's, shared by + // every node running the module (see portInto). + var filled map[string]any + for _, key := range named { + written, ok := env[key].(string) + if !ok || !ofSeat.MatchString(written) { + continue + } + value, err := seatsFilledInto(written, + fmt.Sprintf("%s's container %s sets %s to something that", + module, resource["name"], key), with) + if err != nil { + return err + } + if filled == nil { + filled = map[string]any{} + for k, v := range env { + filled[k] = v + } + } + filled[key] = value + } + if filled != nil { + resource["env"] = filled + } + } + return nil +} + +// seatsFilledInto answers every ${seat:…} in one written value. +// +// A port the seat's holder does not publish on this machine — or a seat nothing on it holds — +// answers with nothing, for the reason the package comment gives. A port that is not one is +// refused: it was written by a person and it is wrong. +func seatsFilledInto(written, where string, with Rendering) (string, error) { + for _, m := range ofSeat.FindAllStringSubmatch(written, -1) { + seat, port := m[1], m[2] + wanted, err := strconv.Atoi(port) + if err != nil || wanted < 1 || wanted > 65535 { + return "", fmt.Errorf("%s says ${seat:%s:%s}, and %s is not a port", where, seat, port, port) + } + answer := "" + if at, known := with.Seats[seat][wanted]; known { + answer = strconv.Itoa(at) + } + written = strings.ReplaceAll(written, m[0], answer) + } + return written, nil +} diff --git a/internal/catalogue/seat_into_test.go b/internal/catalogue/seat_into_test.go new file mode 100644 index 0000000..bb7dc6b --- /dev/null +++ b/internal/catalogue/seat_into_test.go @@ -0,0 +1,216 @@ +package catalogue + +import ( + "os" + "strings" + "testing" +) + +// The control plane's own addresses follow the node's ports (novox/hq 04-ISSUES/102). + +func TestASeatPlaceholderAnswersWhereThisMachinePutTheHolder(t *testing.T) { + control := map[string]any{ + "type": "container", "id": "server", "name": "mesh-controller", + "env": map[string]any{ + "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", + "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", + "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", + "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", + }, + } + with := Rendering{Seats: map[string]map[int]int{ + "mesh-store": {5432: 6852}, "mesh-broker": {5672: 5679, 5671: 5671}, + }} + if err := seatInto(control, "mesh-controller", with); err != nil { + t.Fatal(err) + } + env := control["env"].(map[string]any) + for key, want := range map[string]string{ + "MESH_STORE_INVENTORY_PORT": "6852", + "MESH_BROKER_AMQP_PORT": "5679", + "MESH_BROKER_ADDRESS_PORT": "5671", + "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", + } { + if env[key] != want { + t.Errorf("%s = %v, want %q", key, env[key], want) + } + } +} + +// A seat nothing on this machine holds — or one whose holder the mesh has given no port — answers +// with nothing, so the port genesis wrote into the connection string stands. Not the software's +// own port: on a node given a port at genesis before the store's module exists, that would +// override the right number with the catalogue's. +func TestASeatTheMeshCannotPlaceAnswersWithNothing(t *testing.T) { + control := map[string]any{ + "type": "container", "id": "server", "name": "mesh-controller", + "env": map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"}, + } + if err := seatInto(control, "mesh-controller", Rendering{}); err != nil { + t.Fatal(err) + } + if got := control["env"].(map[string]any)["MESH_STORE_INVENTORY_PORT"]; got != "" { + t.Fatalf("with nothing known, the seat answered %q", got) + } + file := map[string]any{"type": "file", "content": "port=${seat:mesh-store:5432}\n"} + if err := seatInto(file, "x", Rendering{Seats: map[string]map[int]int{"mesh-store": {5433: 1}}}); err != nil { + t.Fatal(err) + } + if got := file["content"]; got != "port=\n" { + t.Fatalf("a port the holder does not publish answered %q", got) + } +} + +func TestASeatPlaceholderNamingNoPortIsRefused(t *testing.T) { + file := map[string]any{"type": "file", "content": "${seat:mesh-store:99999}"} + if err := seatInto(file, "x", Rendering{}); err == nil { + t.Fatal("99999 was accepted as a port") + } +} + +func TestFillingASeatLeavesTheManifestAlone(t *testing.T) { + env := map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"} + manifest := map[string]any{"type": "container", "id": "server", "env": env} + for _, at := range []int{6852, 5432} { + copied := map[string]any{} + for k, v := range manifest { + copied[k] = v + } + with := Rendering{Seats: map[string]map[int]int{"mesh-store": {5432: at}}} + if err := seatInto(copied, "mesh-controller", with); err != nil { + t.Fatal(err) + } + } + if env["MESH_STORE_INVENTORY_PORT"] != "${seat:mesh-store:5432}" { + t.Fatalf("the module's own manifest was edited: %v", env) + } +} + +// **The control plane's own manifest, composed through the whole path.** +// +// The store was given 6852 and the broker's plain port 5679 (the control-node's migration, novox/hq +// 04-ISSUES/102). The control plane's own connections are sealed at genesis with the ports genesis +// wrote; what its container is told beside them is where this machine put the store and the +// broker now, read from the node's settings exactly as every consumer's binding is. +func TestTheControlPlanesOwnAddressesFollowTheNodesPorts(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + m, err := ParseManifest(raw) + if err != nil { + t.Fatalf("the control plane's own manifest does not parse:\n%v", err) + } + // The manifest itself names them now; withSeatPorts is a no-op on it, and this holds it so. + for _, r := range m.Resources { + if r["type"] != "container" { + continue + } + env, _ := r["env"].(map[string]any) + for key, want := range SeatPorts { + if env[key] != want { + t.Errorf("module.json says %s=%v, not %q", key, env[key], want) + } + } + } + m = withSeatPorts(m) + control, err := m.Resolve([]Built{{ + Name: "server", Kind: ArtifactImage, + Reference: ArtifactStoreScheme + "mesh-controller/server@sha256:" + strings.Repeat("c", 64), + }}) + if err != nil { + t.Fatal(err) + } + r := Resolution{Node: "anchor", Modules: []Manifest{control}} + needed := map[string]map[string]string{"mesh-controller": {}} + for name := range m.OwnSecrets { + needed["mesh-controller"][name] = "sealed-" + name + } + out, err := r.Declaration(Rendering{ + Needed: needed, + ArtifactStore: "anchor.internal:5100", + Seats: map[string]map[int]int{ + "mesh-store": {5432: 6852}, + "mesh-broker": {5671: 5671, 5672: 5679, 15672: 15673}, + }, + }) + if err != nil { + t.Fatalf("the control plane does not compose: %v", err) + } + server := fileNamed(out, "mesh-controller.server") + if server == nil { + t.Fatalf("the control plane's container is not in the declaration: %v", out) + } + env, _ := server["env"].(map[string]any) + for key, want := range map[string]string{ + "MESH_STORE_INVENTORY_PORT": "6852", + "MESH_STORE_IDENTITY_PORT": "6852", + "MESH_STORE_LICENCES_PORT": "6852", + "MESH_BROKER_AMQP_PORT": "5679", + "MESH_BROKER_MANAGEMENT_PORT": "15673", + "MESH_BROKER_ADDRESS_PORT": "5671", + } { + if env[key] != want { + t.Errorf("the control plane is told %s=%v; the node put it on %s", key, env[key], want) + } + } + if got := server["image"]; got != "anchor.internal:5100/mesh-controller/server@sha256:"+strings.Repeat("c", 64) { + t.Errorf("the control plane's own image is %v, not routed through the store", got) + } + + // And on a mesh where the foundation is where genesis raised it, nothing is added. + out, err = r.Declaration(Rendering{Needed: needed, ArtifactStore: "anchor.internal:5100"}) + if err != nil { + t.Fatal(err) + } + env, _ = fileNamed(out, "mesh-controller.server")["env"].(map[string]any) + if env["MESH_STORE_INVENTORY_PORT"] != "" || env["MESH_BROKER_AMQP_PORT"] != "" { + t.Errorf("with no settings, the control plane is told %v", env) + } +} + +// SeatPorts is what the control plane's manifest says beside each sealed connection: the port +// this machine put the seat's holder at (novox/hq 04-ISSUES/102). +var SeatPorts = map[string]string{ + "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}", + "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", + "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", + "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", +} + +// withSeatPorts is the control plane's manifest with SeatPorts in its container's environment. +// +// **The manifest lands one commit after the code that fills it**, deliberately: a control plane +// still running the previous build passes `${seat:…}` through unfilled, and the manifest may only +// name the placeholder once every control plane that could compose it knows it. So the test does +// not depend on module.json carrying these yet, and is a no-op once it does. +func withSeatPorts(m Manifest) Manifest { + out := m + out.Resources = nil + for _, r := range m.Resources { + if r["type"] != "container" { + out.Resources = append(out.Resources, r) + continue + } + copied := map[string]any{} + for k, v := range r { + copied[k] = v + } + env := map[string]any{} + if had, ok := r["env"].(map[string]any); ok { + for k, v := range had { + env[k] = v + } + } + for k, v := range SeatPorts { + if _, said := env[k]; !said { + env[k] = v + } + } + copied["env"] = env + out.Resources = append(out.Resources, copied) + } + return out +} diff --git a/internal/envfile/port.go b/internal/envfile/port.go new file mode 100644 index 0000000..e95f878 --- /dev/null +++ b/internal/envfile/port.go @@ -0,0 +1,146 @@ +package envfile + +import ( + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" +) + +// The port a machine put something on, said beside the value that names it. +// +// **An address written down when a port was decided does not follow the port** (novox/hq +// 04-ISSUES/102). The control plane's own store and broker connections are written at genesis — +// full connection strings, password and all — and sealed, so the mesh cannot open them to move the +// port inside. When the node's settings move that port, every consumer's binding follows and the +// control plane's own connection does not: it goes on dialling the number genesis wrote, and the +// mesh is headless. +// +// So a setting has a third twin. `NAME_FILE` says where the value is; `NAME_PORT` says which port +// this machine put the thing at, composed into the control plane's environment from the node's +// settings exactly as a consumer's binding is. The value's own port is what stands when the twin +// says nothing — a mesh whose foundation is still where genesis raised it needs no override, and +// an empty answer is the mesh saying it has nothing to add, not the mesh saying zero. +// +// Only the port. The host inside the value is the machine's own loopback, or the broker's public +// name — a fact of the genesis, not of any node's settings — and the mesh has no better opinion +// of it. What moves under ADR 0100 is the port. + +// PortVar is the twin saying which port this machine put the named thing at. +func PortVar(name string) string { return name + "_PORT" } + +// Port is what NAME_PORT says, checked to be a port, or "" when it says nothing. +// +// Blank counts as unset, as it does for every other variable here: a container given an empty +// string was given nothing, and refusing it as ambiguous would turn an omission into a puzzle. +func Port(name string) (string, error) { + raw := strings.TrimSpace(os.Getenv(PortVar(name))) + if raw == "" { + return "", nil + } + if unfilled.MatchString(raw) { + // **A placeholder the mesh never filled, and this is the one place it must not be a + // fault.** The control plane composes its own declaration, so a manifest naming + // `${seat:…}` reaches a control plane one build older than the manifest — one that does + // not know the placeholder and passes it through as the value. Refusing it here would + // leave every command and the daemon unable to open a store: headless, on the machine + // that cannot be repaired through the mesh (novox/hq 04-ISSUES/102). So it is what an + // empty answer is — nothing said, the sealed value stands — and it is said aloud, because + // a manifest ahead of its binary is worth a line on stderr and not worth an outage. + fmt.Fprintf(os.Stderr, "%s is %q, a placeholder nothing filled in — ignored; the port in "+ + "%s stands. The control plane composing this declaration is older than the manifest "+ + "naming it: rebuild and push it\n", PortVar(name), raw, name) + return "", nil + } + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > 65535 { + return "", fmt.Errorf("%s is %q, which is not a port", PortVar(name), raw) + } + return strconv.Itoa(n), nil +} + +// Placed is Value, with its port moved to where NAME_PORT says this machine put it. +func Placed(name string) (string, error) { + value, err := Value(name) + if err != nil { + return "", err + } + port, err := Port(name) + if err != nil || port == "" { + return value, err + } + placed, err := WithPort(value, port) + if err != nil { + // Where it came from is said; what it says is not. The value is a connection string with + // a password in it, and every error here is written on the assumption it will be logged. + return "", fmt.Errorf("%s names a port to move %s to, and %s could not be read as "+ + "something with a port in it: %w", PortVar(name), name, name, err) + } + return placed, nil +} + +// keywordPort is `port=…` in a `key=value` connection string. +var keywordPort = regexp.MustCompile(`(^|\s)port=\S*`) + +// unfilled is a value that is still a placeholder — `${…}` — rather than something a person or the +// mesh wrote as a port. +var unfilled = regexp.MustCompile(`^\$\{[^}]*\}$`) + +// WithPort is the value with its port replaced by `port`. +// +// Three shapes, because the control plane's settings come in three: a URL +// (`scheme://[user:password@]host[:port][/…]`), a bare `host[:port]` (the broker's address) and +// a `key=value` connection string (`host=… port=…`), which is what the store's driver accepts +// beside a URL. An IPv6 host stays in its brackets. Nothing here parses the URL properly — a +// password with a character a URL parser dislikes is still a working connection string, and +// refusing it would refuse a value that has been dialling fine since genesis. +func WithPort(value, port string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", fmt.Errorf("the value is empty") + } + if scheme, rest, isURL := strings.Cut(value, "://"); isURL { + // **The password may hold any of `/ ? # @`, so the host begins after the LAST `@`**, and + // the path only after that. Cutting at the first `/` would take a password's slash for the + // path's and put the new port on the user name — a connection string that dials the wrong + // host without a word. Genesis makes passwords that cannot do this; an accepted one can. + // The connection strings this reads — a store's, a broker's, a management API's — carry + // no `@` after their userinfo, which is what makes the last one the boundary. + userinfo, hostAndTail := "", rest + if at := strings.LastIndex(rest, "@"); at >= 0 { + userinfo, hostAndTail = rest[:at+1], rest[at+1:] + } + authority, tail := hostAndTail, "" + if end := strings.IndexAny(hostAndTail, "/?#"); end >= 0 { + authority, tail = hostAndTail[:end], hostAndTail[end:] + } + host, err := hostWithPort(authority, port) + if err != nil { + return "", err + } + return scheme + "://" + userinfo + host + tail, nil + } + if strings.Contains(value, "=") && !strings.ContainsAny(value, "/") { + if keywordPort.MatchString(value) { + return keywordPort.ReplaceAllString(value, "${1}port="+port), nil + } + return value + " port=" + port, nil + } + return hostWithPort(value, port) +} + +// hostWithPort is `host[:port]` with the port set, brackets kept around an IPv6 host. +func hostWithPort(authority, port string) (string, error) { + if authority == "" { + return "", fmt.Errorf("there is no host to put a port on") + } + host, _, err := net.SplitHostPort(authority) + if err != nil { + // No port yet. A bracketed IPv6 host without a port is a shape SplitHostPort refuses, and + // a bare one carries colons of its own — both are a host, not an error. + host = strings.TrimSuffix(strings.TrimPrefix(authority, "["), "]") + } + return net.JoinHostPort(host, port), nil +} diff --git a/internal/envfile/port_test.go b/internal/envfile/port_test.go new file mode 100644 index 0000000..bf5c485 --- /dev/null +++ b/internal/envfile/port_test.go @@ -0,0 +1,78 @@ +package envfile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The control plane's own connections follow the port the node moved (novox/hq 04-ISSUES/102). + +func TestAPortTwinMovesTheValuesPort(t *testing.T) { + cases := map[string]string{ + "postgres://mesh:s3cret@127.0.0.1:5432/inventory?sslmode=disable": "postgres://mesh:s3cret@127.0.0.1:6852/inventory?sslmode=disable", + "postgres://mesh:s3cret@127.0.0.1/inventory": "postgres://mesh:s3cret@127.0.0.1:6852/inventory", + "amqp://control:p%40ss@127.0.0.1:5672/": "amqp://control:p%40ss@127.0.0.1:6852/", + "amqp://control:p@ss:with@127.0.0.1:5672/": "amqp://control:p@ss:with@127.0.0.1:6852/", + "http://guest:guest@127.0.0.1:15672": "http://guest:guest@127.0.0.1:6852", + "postgres://mesh:a/b?c#d@e@127.0.0.1:5432/inventory": "postgres://mesh:a/b?c#d@e@127.0.0.1:6852/inventory", + "http://[::1]:15672/api": "http://[::1]:6852/api", + "broker.example:5671": "broker.example:6852", + "broker.example": "broker.example:6852", + "host=127.0.0.1 port=5432 dbname=inventory": "host=127.0.0.1 port=6852 dbname=inventory", + "host=127.0.0.1 dbname=inventory": "host=127.0.0.1 dbname=inventory port=6852", + } + for value, want := range cases { + got, err := WithPort(value, "6852") + if err != nil || got != want { + t.Errorf("WithPort(%q) = %q, %v; want %q", value, got, err, want) + } + } +} + +func TestPlacedLeavesTheValueAloneWhenNothingSaysAPort(t *testing.T) { + path := filepath.Join(t.TempDir(), "value") + if err := os.WriteFile(path, []byte("postgres://m:p@127.0.0.1:5432/inventory\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MESH_P_FILE", path) + t.Setenv("MESH_P_PORT", "") + got, err := Placed("MESH_P") + if err != nil || got != "postgres://m:p@127.0.0.1:5432/inventory" { + t.Fatalf("got %q, %v", got, err) + } + t.Setenv("MESH_P_PORT", " 6852 ") + got, err = Placed("MESH_P") + if err != nil || got != "postgres://m:p@127.0.0.1:6852/inventory" { + t.Fatalf("got %q, %v", got, err) + } +} + +func TestAPortTwinThatIsNotAPortIsRefusedWithoutQuotingTheValue(t *testing.T) { + t.Setenv("MESH_Q", "postgres://m:hunter2@127.0.0.1:5432/inventory") + t.Setenv("MESH_Q_PORT", "many") + _, err := Placed("MESH_Q") + if err == nil { + t.Fatal("a port that is not a number was accepted") + } + if strings.Contains(err.Error(), "hunter2") { + t.Fatalf("the value was quoted back: %v", err) + } + t.Setenv("MESH_Q", "") + t.Setenv("MESH_Q_PORT", "6852") + if _, err := Placed("MESH_Q"); err == nil { + t.Fatal("a port with no value to put it on was accepted") + } +} + +// A placeholder nothing filled is nothing said, not a fault: the control plane composing the +// declaration may be one build behind the manifest, and refusing would leave it headless. +func TestAnUnfilledPlaceholderIsNothingSaid(t *testing.T) { + t.Setenv("MESH_R", "postgres://m:p@127.0.0.1:5432/inventory") + t.Setenv("MESH_R_PORT", "${seat:mesh-store:5432}") + got, err := Placed("MESH_R") + if err != nil || got != "postgres://m:p@127.0.0.1:5432/inventory" { + t.Fatalf("an unfilled placeholder was not ignored: %q, %v", got, err) + } +} diff --git a/internal/link/serve.go b/internal/link/serve.go index 3bc2d76..925c237 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -146,8 +146,12 @@ func (s *Server) Answers(r Replayer) error { } // Connect opens the control plane's own connection to the broker. +// +// On the port MESH_BROKER_AMQP_PORT names when the node's settings moved the broker (novox/hq +// 04-ISSUES/102) — the URL is genesis's, sealed, and its port is the one thing in it the node may +// have moved since. func Connect(enroller Enroller, listener Listener) (*Server, error) { - url, err := envfile.Value(AMQPVar) + url, err := envfile.Placed(AMQPVar) if err != nil { return nil, err } diff --git a/internal/store/manifest_test.go b/internal/store/manifest_test.go new file mode 100644 index 0000000..1a47e6a --- /dev/null +++ b/internal/store/manifest_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "encoding/json" + "os" + "testing" +) + +// **The manifest's placeholder, unfilled, reaches a store reader and changes nothing.** +// +// The control plane composes its own declaration, so the manifest naming `${seat:…}` can be +// composed by a control plane one build older than it — one that passes the literal through as +// the value. That is the state of the live control-node between this manifest landing and its +// next build being pushed, and a reader that refused the literal would leave it headless +// (novox/hq 04-ISSUES/102, finding F1). So the reader is held to ignoring exactly what +// module.json says, not a placeholder shaped like it. +func TestTheManifestsOwnPlaceholderUnfilledLeavesTheStoreWhereTheFileSays(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []struct { + Type string `json:"type"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var written string + for _, r := range m.Resources { + if r.Type == "container" { + written = r.Env["MESH_STORE_INVENTORY_PORT"] + } + } + if written == "" { + t.Fatal("module.json no longer names MESH_STORE_INVENTORY_PORT") + } + + alone(t) + t.Setenv(Variable(example), dsn) + t.Setenv(PortVariable(example), written) + opened, err := Open(t.Context(), example) + if err != nil { + t.Fatalf("the unfilled placeholder was refused, which is a headless control plane: %v", err) + } + defer opened.Close() + if got := opened.Pool().Config().ConnConfig.Port; got != 5432 { + t.Fatalf("the store is on %d; with the placeholder unfilled, the file's port stands", got) + } +} diff --git a/internal/store/settings_test.go b/internal/store/settings_test.go index ece30c5..f9fcd5e 100644 --- a/internal/store/settings_test.go +++ b/internal/store/settings_test.go @@ -213,3 +213,55 @@ func mustNotLeak(t *testing.T, err error) { t.Fatalf("the password is in the error: %v", err) } } + +// The node moved the store, and the control plane's own connection follows (novox/hq 04-ISSUES/102). +// +// The connection string is what genesis wrote, port and all; the port twin is what the node's +// settings say now. The pool's configuration is the one place both meet. +func TestThePortTwinMovesTheStoresPort(t *testing.T) { + alone(t) + t.Setenv(PortVariable(example), "") + path := filepath.Join(t.TempDir(), "inventory") + if err := os.WriteFile(path, []byte(dsn+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(FileVariable(example), path) + + opened, err := Open(t.Context(), example) + if err != nil { + t.Fatal(err) + } + if got := opened.Pool().Config().ConnConfig.Port; got != 5432 { + t.Fatalf("with nothing said, the store is on %d rather than what the file says", got) + } + opened.Close() + + t.Setenv(PortVariable(example), "6852") + opened, err = Open(t.Context(), example) + if err != nil { + t.Fatalf("a moved port was refused: %v", err) + } + defer opened.Close() + if got := opened.Pool().Config().ConnConfig.Port; got != 6852 { + t.Fatalf("the store is on %d, and the node put it on 6852", got) + } + if got := opened.Pool().Config().ConnConfig.Password; got != "s3cret-in-here" { + t.Fatalf("moving the port changed the password to %q", got) + } +} + +func TestAPortTwinThatIsNotAPortNamesItselfAndNotTheSecret(t *testing.T) { + alone(t) + t.Setenv(Variable(example), dsn) + t.Setenv(PortVariable(example), "six") + _, err := Open(t.Context(), example) + if err == nil { + t.Fatal("a port that is not a number was accepted") + } + if !strings.Contains(err.Error(), PortVariable(example)) { + t.Errorf("the error does not name the variable: %v", err) + } + if strings.Contains(err.Error(), "s3cret") { + t.Errorf("the error quotes the password: %v", err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 9d8ab7b..d672bfb 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -25,6 +25,8 @@ import ( "time" "github.com/jackc/pgx/v5/pgxpool" + + "github.com/novox/mesh-controller/internal/envfile" ) // contextName is what a context may be called. @@ -67,6 +69,17 @@ func Variable(context string) string { // raises the first one. func FileVariable(context string) string { return Variable(context) + "_FILE" } +// PortVariable is the environment variable naming the PORT this machine put the store at — the +// third twin, beside the value and the file. +// +// **The connection string is sealed and the port inside it is genesis's** (novox/hq 04-ISSUES/102). +// The control plane cannot open its own store secret to move the port, and a node's settings can +// move the store (ADR 0100: the foundation's ports are the node's). Every consumer's binding +// followed that setting; the control plane's own connection did not, and the mesh was headless. So +// the port is composed into the control plane's environment from the node's settings, exactly as a +// consumer's binding is, and the connection string's own port stands only when this says nothing. +func PortVariable(context string) string { return envfile.PortVar(Variable(context)) } + // Database is what a context's database is called. // // Named after the context, so that a person looking at a PostgreSQL server can see which @@ -85,7 +98,7 @@ func Open(ctx context.Context, name string) (*Store, error) { "name, so it must be lower-case letters and digits, starting with a letter", name) } - dsn, from, err := settingsFor(name) + dsn, from, err := placed(name) if err != nil { return nil, err } @@ -169,6 +182,29 @@ func settingsFor(name string) (dsn, from string, err error) { return direct, Variable(name), nil } +// placed is a context's connection string with its port moved to where this machine put the +// store, when the node's settings say so (PortVariable), and as it was otherwise. +func placed(name string) (dsn, from string, err error) { + dsn, from, err = settingsFor(name) + if err != nil { + return "", "", err + } + port, err := envfile.Port(Variable(name)) + if err != nil || port == "" { + return dsn, from, err + } + moved, err := envfile.WithPort(dsn, port) + if err != nil { + // The variable and the port are named; the connection string is not, for the same reason + // as everywhere else in this file. + return "", "", fmt.Errorf( + "%s says this machine put the %s store on port %s, and the connection settings in %s "+ + "could not be read as something with a port in them: %w", + PortVariable(name), name, port, from, err) + } + return moved, from + ", on port " + port + " as " + PortVariable(name) + " says", nil +} + // Context is which context this store belongs to. func (s *Store) Context() string { return s.context } diff --git a/module.json b/module.json index dbb7f0c..936ca87 100644 --- a/module.json +++ b/module.json @@ -42,7 +42,13 @@ "MESH_STORE_LICENCES_FILE": "/run/secrets/licences", "MESH_BROKER_AMQP_FILE": "/run/secrets/broker", "MESH_BROKER_MANAGEMENT_FILE": "/run/secrets/broker-management", - "MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address" + "MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address", + "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}", + "MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}", + "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", + "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", + "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}" }, "volumes": [ "mesh-broker-tls:/broker-tls:ro",