An address is read from the node's settings where it is used, never recorded with a port
Three readers did not follow a moved foundation port (novox/hq 04-ISSUES/102),
and each took the control-node down in its own way: the control plane's own
store and broker connections, sealed at genesis with the port inside; and every
build the mesh ever recorded, kept as `<registry>:<port>/<module>/<artifact>@…`.
The control plane cannot open its own sealed connections to move a port, and it
cannot bind the store as a consumer would — a binding mints a credential. So its
settings get a third twin, `NAME_PORT`, read on top of the sealed value by the
store, the broker, the management API and the bus connection, and filled into
its container by a placeholder that names a seat, `${seat:mesh-store:5432}`,
from the node's given or mesh-assigned ports — never the manifest's number, and
empty when the mesh has nothing to add, so what genesis wrote stands. A value
that is still a placeholder is nothing said, aloud: the manifest naming it lands
in the next commit, once every control plane that composes it knows it.
A build is now recorded by digest and path — `artifact-store://<module>/<artifact>@…`
— and the store's address is composed in where a reference is used: the
declaration, the trust file, the bases a build is handed, a replay to the
catalogue. Over the network as `<node>.internal:<port>`; on the store's own node
before any network exists — every genesis push before its "network" step — by
loopback. A reference recorded before this, with an address, is re-routed the
same way when the mesh built it. The trust file and every provider's address
come from one derivation: the node's given port, over the mesh's assignment,
over the manifest's number.
novox/hq 04-ISSUES/102
This commit is contained in:
@@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// An address is read from the node's settings where it is used, never recorded with a port
|
||||
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
||||
|
||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
|
||||
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
|
||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
manifest, _ := json.Marshal(map[string]any{
|
||||
"module": "gitea", "version": "1",
|
||||
"resources": []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
||||
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
||||
"image": "valkey/valkey@" + aDigest},
|
||||
},
|
||||
})
|
||||
kept := buildFrom(link.BuildResult{
|
||||
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
||||
Manifest: manifest,
|
||||
Made: []link.MadeArtifact{
|
||||
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
},
|
||||
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
||||
})
|
||||
if kept.Module != "gitea" {
|
||||
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
||||
}
|
||||
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
||||
}
|
||||
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
||||
}
|
||||
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the recorded manifest's image is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the recorded manifest's archive is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
||||
t.Errorf("an image the build did not make was rewritten: %v", got)
|
||||
}
|
||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||
}
|
||||
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
|
||||
}
|
||||
}
|
||||
|
||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||
// distribution module does, so a node may be given another number for it.
|
||||
func aStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "distribution", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
||||
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
||||
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
||||
}
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
||||
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
seatPorts := map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
}
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for k, v := range r {
|
||||
copied[k] = v
|
||||
}
|
||||
env := map[string]any{}
|
||||
if had, ok := r["env"].(map[string]any); ok {
|
||||
for k, v := range had {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range seatPorts {
|
||||
if _, said := env[k]; !said {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
copied["env"] = env
|
||||
out.Resources = append(out.Resources, copied)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,11 @@ func buildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// buildFrom turns what a builder said into what the mesh keeps.
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||
// reference and composes the store's address back in where a reference is used. `against` is kept
|
||||
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
|
||||
func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept := inventory.Build{
|
||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||
@@ -77,16 +82,21 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path, Manifest: result.Manifest, Against: result.Against,
|
||||
Path: result.Path, Against: result.Against,
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
announced = append(announced, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
||||
})
|
||||
}
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(result.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(result.Manifest); err == nil {
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
@@ -378,7 +388,8 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -388,13 +399,15 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
|
||||
for _, made := range result.Made {
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||
}
|
||||
|
||||
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
|
||||
// second thing to disagree about what a manifest is.
|
||||
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
|
||||
// holds references by digest and path and every declaration composes the store's address in.
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
@@ -481,6 +494,9 @@ type answers struct {
|
||||
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
||||
// than a build command refusing to start because a query did not run. So the store not opening is
|
||||
// reported and the build goes ahead without it.
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -494,5 +510,18 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
return held
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
||||
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
||||
return held
|
||||
}
|
||||
if address == "" {
|
||||
return held
|
||||
}
|
||||
routed := make(map[string]string, len(held))
|
||||
for repository, reference := range held {
|
||||
routed[repository] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
@@ -440,8 +440,11 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
|
||||
// module providing it is assigned to a machine that is on the private network.
|
||||
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
|
||||
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
|
||||
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
|
||||
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
|
||||
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
|
||||
//
|
||||
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
|
||||
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
|
||||
@@ -454,29 +457,87 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
|
||||
}
|
||||
providers := map[string]string{} // module -> served port
|
||||
providers := map[string]catalogue.Manifest{}
|
||||
for name, m := range shelf {
|
||||
served, offers := m.Serves[catalogue.ArtifactStoreProvision]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
if p, ok := served["port"]; ok {
|
||||
providers[name] = fmt.Sprintf("%v", p)
|
||||
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
|
||||
providers[name] = m
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
// In a stated order, so two machines offering it would always answer the same one.
|
||||
machines := make([]string, 0, len(on))
|
||||
for machine := range on {
|
||||
machines = append(machines, machine)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
for _, machine := range machines {
|
||||
assigned, err := inv.Assigned(ctx, machine)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
|
||||
}
|
||||
for _, a := range assigned {
|
||||
if p, ok := providers[a]; ok {
|
||||
return machine, p, true, nil
|
||||
m, offers := providers[a]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
if p, ok := serves["port"]; ok {
|
||||
return machine, fmt.Sprintf("%v", p), true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
|
||||
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
|
||||
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
|
||||
// node that holds the store itself, and "" for any other. The address composed into every
|
||||
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
|
||||
// of those is built and pushed to a node that is on no network, and the store is on that same
|
||||
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
|
||||
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
|
||||
// address genesis itself reaches the store by.
|
||||
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
on := map[string]bool{}
|
||||
for name := range onNetwork {
|
||||
on[name] = true
|
||||
}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found {
|
||||
return overlay.InternalName(node) + ":" + port, nil
|
||||
}
|
||||
holder, port, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found || holder != forNode {
|
||||
return "", err
|
||||
}
|
||||
return "127.0.0.1:" + port, nil
|
||||
}
|
||||
|
||||
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
|
||||
// off the network, and the port that node put it on.
|
||||
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n.Name] = true
|
||||
}
|
||||
return artifactStoreOnNetwork(ctx, inv, all)
|
||||
}
|
||||
|
||||
+138
-20
@@ -229,28 +229,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
// What that module says a consumer needs to know, with that node's settings on
|
||||
// it: a port somebody moved on the provider is a port its consumers must be told
|
||||
// about, and the two coming from different places is how they come to disagree.
|
||||
assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
|
||||
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
// A port that node was given is where its consumers reach it (novox/hq ADR
|
||||
// 0100). Unreadable given ports are that node's refusal to report, not this one's.
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
for wanted, at := range given {
|
||||
assigned[wanted] = at
|
||||
}
|
||||
}
|
||||
serves := catalogue.ServedOn(m, name, assigned)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
}
|
||||
offered[name] = append(offered[name], catalogue.Provider{
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves})
|
||||
}
|
||||
@@ -499,6 +481,22 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// The artifact store as this node reaches it now — the address every image and archive the
|
||||
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
||||
// built, so a reference recorded with an address before that is re-routed too.
|
||||
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
built := make(map[string]bool, len(held))
|
||||
for repository := range held {
|
||||
built[repository] = true
|
||||
}
|
||||
|
||||
// And every machine's name, so a container can reach one. The same set that writes the
|
||||
// machine's own hosts file — one reading, so a container and its machine cannot disagree
|
||||
// about where another machine is.
|
||||
@@ -566,11 +564,18 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
taken[m] = true
|
||||
}
|
||||
}
|
||||
// Where this node put the foundation's servers, for the control plane's own connections
|
||||
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
|
||||
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
|
||||
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
|
||||
Given: given, Taken: taken,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
@@ -967,6 +972,119 @@ func managerPublicKeyFor(
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
}
|
||||
|
||||
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
|
||||
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
|
||||
// assigned over the manifest's own, settled with the node's settings layers.
|
||||
//
|
||||
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
|
||||
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
||||
// given ports are that node's refusal to report, not this reader's.
|
||||
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest, provision string) (map[string]any, error) {
|
||||
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
serves := catalogue.ServedOn(m, provision, ports)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return serves, nil
|
||||
}
|
||||
|
||||
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
||||
// node's settings layers for the module, read once for both.
|
||||
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
|
||||
ports, err := portsOn(ctx, inv, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
for wanted, at := range given {
|
||||
ports[wanted] = at
|
||||
}
|
||||
}
|
||||
return ports, layers, nil
|
||||
}
|
||||
|
||||
// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the
|
||||
// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// Read for every module in the catalogue that claims a seat, in this node's set or not: the store
|
||||
// and the broker are given their ports at genesis, as settings on a module that may be registered
|
||||
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
|
||||
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
|
||||
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||
node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) {
|
||||
assigned := map[string]bool{}
|
||||
for _, m := range inSet {
|
||||
assigned[m.Module] = true
|
||||
}
|
||||
names := make([]string, 0, len(shelf))
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
seats := map[string]map[int]int{}
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
var claims []string
|
||||
for _, c := range m.Claims {
|
||||
if c.At() == catalogue.ScopeMesh {
|
||||
claims = append(claims, c.Name)
|
||||
}
|
||||
}
|
||||
if len(claims) == 0 {
|
||||
continue
|
||||
}
|
||||
// **Only a port the node was given or the mesh assigned — never the manifest's own
|
||||
// number.** The sealed value the answer sits beside carries the port genesis wrote, which
|
||||
// on a given-port node is the predecessor's; a manifest's long-form mapping is the
|
||||
// catalogue's default, and answering with it would override the right number with one
|
||||
// the mesh never checked (the contract in seat_into.go). And on an adopted node a holder
|
||||
// assigned but not yet taken is the found container, on the ports it was found with, not
|
||||
// the declaration's — so its mesh-assigned ports do not count there either; a given port
|
||||
// does, because a given port is the found one by construction (ADR 0100).
|
||||
ports, _, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if adopted && !taken[name] {
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ports = map[int]int{}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
ports = given
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, seat := range claims {
|
||||
if seats[seat] == nil {
|
||||
seats[seat] = map[int]int{}
|
||||
}
|
||||
for wanted, at := range ports {
|
||||
if _, said := seats[seat][wanted]; said && !assigned[name] {
|
||||
continue
|
||||
}
|
||||
seats[seat][wanted] = at
|
||||
}
|
||||
}
|
||||
}
|
||||
return seats, nil
|
||||
}
|
||||
|
||||
// portsOn is one module's assignments on one machine, by the port the software uses.
|
||||
func portsOn(
|
||||
ctx context.Context, inv *inventory.Inventory, node, module string,
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a build is recorded as, and what it is announced and handed on as.
|
||||
//
|
||||
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
|
||||
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — and the manifest with those references in it. The mesh records the digest and the path
|
||||
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
|
||||
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
|
||||
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
|
||||
// rebuild of everything the mesh has ever built.
|
||||
|
||||
// recordedManifest is a build's manifest with the store's address taken off every reference the
|
||||
// build itself made. Other references — an image a module runs from a public registry — are what
|
||||
// they were, which is why this rewrites only what `made` names rather than everything that looks
|
||||
// like an address.
|
||||
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
|
||||
announced := map[string]bool{}
|
||||
for _, a := range made {
|
||||
announced[a.Reference] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !announced[ref] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Recorded(ref), true
|
||||
})
|
||||
}
|
||||
|
||||
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
|
||||
// composed into every reference the build made — recorded either way, before or after references
|
||||
// were kept without their address.
|
||||
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
|
||||
recorded := map[string]bool{}
|
||||
for _, a := range made {
|
||||
recorded[catalogue.Recorded(a.Reference)] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !recorded[catalogue.Recorded(ref)] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Rerouted(ref, address), true
|
||||
})
|
||||
}
|
||||
|
||||
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
|
||||
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
|
||||
// is, is the parser's to say, and it says so with a better sentence than anything here would.
|
||||
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
var manifest map[string]any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return raw
|
||||
}
|
||||
resources, _ := manifest["resources"].([]any)
|
||||
changed := false
|
||||
for _, r := range resources {
|
||||
resource, ok := r.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, key := range []string{"image", "source"} {
|
||||
if written, ok := resource[key].(string); ok {
|
||||
if rewritten, did := rewrite(written); did && rewritten != written {
|
||||
resource[key] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// routedArtifacts is a build's artifacts as something can fetch them now.
|
||||
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
|
||||
if address == "" {
|
||||
return made
|
||||
}
|
||||
out := make([]inventory.Artifact, 0, len(made))
|
||||
for _, a := range made {
|
||||
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
|
||||
Reference: catalogue.Rerouted(a.Reference, address)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
|
||||
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
|
||||
// store's own node: loopback when nothing is on the network yet.
|
||||
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if forNode == "" {
|
||||
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
|
||||
return "", err
|
||||
} else if found {
|
||||
forNode = holder
|
||||
}
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, forNode)
|
||||
}
|
||||
|
||||
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
|
||||
// when there is one, else loopback on the store's own node — when that node also holds a module
|
||||
// requiring the store, which is what a builder is (genesis: one node holds both).
|
||||
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holder, _, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found {
|
||||
return "", err
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
besideIt := false
|
||||
for _, a := range assigned {
|
||||
for _, r := range shelf[a].Requires {
|
||||
if r == catalogue.ArtifactStoreProvision {
|
||||
besideIt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !besideIt {
|
||||
holder = ""
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, holder)
|
||||
}
|
||||
@@ -173,11 +173,21 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
|
||||
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
|
||||
// store the catalogue runs on, and the catalogue itself.
|
||||
//
|
||||
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
|
||||
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
|
||||
// the store's address, so a replay composes the address back in — the store's address as the
|
||||
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
|
||||
// store on the network yet, the recorded form goes as it is.
|
||||
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
|
||||
builds, err := f.open.inventory.Announceable(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]link.Announcement, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
a := link.Announcement{
|
||||
@@ -186,8 +196,11 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
}
|
||||
if len(b.Manifest) > 0 {
|
||||
a.Manifest = b.Manifest
|
||||
if address != "" {
|
||||
a.Manifest = routedManifest(b.Manifest, b.Made, address)
|
||||
}
|
||||
}
|
||||
for _, made := range b.Made {
|
||||
for _, made := range routedArtifacts(b.Made, address) {
|
||||
a.Made = append(a.Made, link.MadeArtifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user