package main import ( "encoding/json" "os" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" "github.com/novox/mesh-controller/internal/link" ) // An address is read from the node's settings where it is used, never recorded with a port // (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here. var aDigest = "sha256:" + strings.Repeat("e", 64) // **A build is recorded by digest and path**, whatever address the builder pushed to — what it // made and what it stood on both; an image the module runs from elsewhere is left where it says. func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) { manifest, _ := json.Marshal(map[string]any{ "module": "gitea", "version": "1", "resources": []map[string]any{ {"id": "server", "type": "container", "name": "mesh-gitea", "image": "anchor.internal:5100/gitea/server@" + aDigest}, {"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest, "source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest}, {"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + aDigest}, }, }) kept := buildFrom(link.BuildResult{ ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop", Manifest: manifest, Made: []link.MadeArtifact{ {Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest}, {Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest}, }, Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest}, }) if kept.Module != "gitea" { t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module) } if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest { t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference) } if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest { t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference) } recorded, err := catalogue.ParseManifest(kept.Manifest) if err != nil { t.Fatal(err) } if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest { t.Errorf("the recorded manifest's image is %v", got) } if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest { t.Errorf("the recorded manifest's archive is %v", got) } if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest { t.Errorf("an image the build did not make was rewritten: %v", got) } if strings.Contains(string(kept.Manifest), "anchor.internal:5100") { t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest) } // What the build stood on is an edge to another module's artifact, and it is recorded the way // that artifact is: by path in the store, so the edge still names the same thing when the // store answers at another address. if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest { t.Errorf("what the build stood on was recorded by address: %v", kept.Against) } } // aStore is a module offering the artifact store on 5000, published the long way as the // distribution module does, so a node may be given another number for it. func aStore() catalogue.Manifest { return catalogue.Manifest{Module: "distribution", Version: "1", Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}}, Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}}, Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}}, Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry", "ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}} } // **The trust a machine writes for the store, and the address every built image is fetched // through, say the port the node gave the store** — not the catalogue's number. func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { open := aMesh(t) ctx := t.Context() register(t, open, aStore()) if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { t.Fatal(err) } if err := open.inventory.SetSettings(ctx, "anchor", "distribution", map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { t.Fatal(err) } // A module the mesh built, recorded by digest and path, running on the other machine. if err := open.inventory.RecordBuild(ctx, inventory.Build{ ID: "b1", Repository: "r", Module: "app", Commit: "abc", Made: []inventory.Artifact{{Name: "server", Kind: "image", Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, }); err != nil { t.Fatal(err) } register(t, open, catalogue.Manifest{Module: "app", Version: "1", Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) if _, err := assign(ctx, open, "laptop", "app"); err != nil { t.Fatal(err) } on := map[string]bool{"anchor": true, "laptop": true} node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) if err != nil || !found || node != "anchor" || port != "5101" { t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) } var trust string for _, r := range composed(t, open, "laptop").Resources { if r["path"] == "/etc/docker/daemon.json" { trust, _ = r["content"].(string) } if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { t.Errorf("the image the mesh built is fetched as %v", r["image"]) } } if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) } // And a replay to the catalogue says where the store is now. announced, err := following{open}.Announceable(ctx) if err != nil { t.Fatal(err) } if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { t.Fatalf("the replay announces %+v", announced) } } // **The control plane's own connections say the port the node gave the store and the broker.** // // Composed from the control plane's own manifest against a real inventory: the store's module is // given 6852 on this node the way genesis or an operator gives it, and the control plane's // container is told so beside the sealed connection genesis wrote. func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { open := aMesh(t) ctx := t.Context() raw, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) } m, err := catalogue.ParseManifest(raw) if err != nil { t.Fatal(err) } control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage, Reference: "registry.example/control@" + aDigest}}) if err != nil { t.Fatal(err) } register(t, open, control) register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, {Port: 5672, From: catalogue.FromMesh}}, Guards: []int{15672}, Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { t.Fatal(err) } // The store's module is registered and given its port, and NOT assigned: the state genesis // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). if err := open.inventory.SetSettings(ctx, "anchor", "postgres", map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { t.Fatal(err) } if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { t.Fatal(err) } if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { t.Fatal(err) } var env map[string]any for _, r := range composed(t, open, "anchor").Resources { if r["id"] == "mesh-controller.server" { env, _ = r["env"].(map[string]any) } } if env == nil { t.Fatal("the control plane's container is not in its own node's declaration") } for key, want := range map[string]string{ "MESH_STORE_INVENTORY_PORT": "6852", "MESH_STORE_IDENTITY_PORT": "6852", "MESH_STORE_LICENCES_PORT": "6852", "MESH_BROKER_AMQP_PORT": "5679", // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, // because the sealed value beside it carries the port genesis wrote (finding F3). "MESH_BROKER_ADDRESS_PORT": "", "MESH_BROKER_MANAGEMENT_PORT": "", } { if env[key] != want { t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) } } } // withSeatPorts is the control plane's manifest with the seat placeholders in its environment — // added here until module.json carries them (the manifest lands one commit after the code that // fills it, so a control plane one build behind never sees a placeholder it cannot fill). func withSeatPorts(m catalogue.Manifest) catalogue.Manifest { seatPorts := map[string]string{ "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", "MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}", "MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}", "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", } out := m out.Resources = nil for _, r := range m.Resources { if r["type"] != "container" { out.Resources = append(out.Resources, r) continue } copied := map[string]any{} for k, v := range r { copied[k] = v } env := map[string]any{} if had, ok := r["env"].(map[string]any); ok { for k, v := range had { env[k] = v } } for k, v := range seatPorts { if _, said := env[k]; !said { env[k] = v } } copied["env"] = env out.Resources = append(out.Resources, copied) } return out } // aLoneNode is one capable machine with nothing placed on any network — the control-node during // genesis, before the "network" step, which is after the store, the broker, the vault and the // catalogue have each been built and pushed (finding F2). func aLoneNode(t *testing.T) *stores { t.Helper() inventory.ForTest(t) licences.ForTest(t) open, err := openStores(t.Context()) if err != nil { t.Fatal(err) } t.Cleanup(open.Close) for _, m := range provided { if err := open.inventory.Provide(t.Context(), m); err != nil { t.Fatal(err) } } record, err := open.inventory.AddNode(t.Context(), "anchor") if err != nil { t.Fatal(err) } reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ {"name": "container-runtime", "present": true}}}) var profile map[string]any _ = json.Unmarshal(reported, &profile) if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { t.Fatal(err) } if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { t.Fatal(err) } return open } // **Before the network exists, the store's own node reaches it by loopback** — never refused, // never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to // a node on no network, and builds the catalogue on a base it must be able to pull. func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { open := aLoneNode(t) ctx := t.Context() register(t, open, aStore()) register(t, open, catalogue.Manifest{Module: "builder", Version: "1", Requires: []string{catalogue.ArtifactStoreProvision}, Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", "image": "registry.example/mesh-builder@" + aDigest}}}) if err := open.inventory.RecordBuild(ctx, inventory.Build{ ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", Made: []inventory.Artifact{{Name: "runtime", Kind: "image", Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, }); err != nil { t.Fatal(err) } register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) for _, module := range []string{"distribution", "builder", "postgres"} { if _, err := assign(ctx, open, "anchor", module); err != nil { t.Fatal(err) } } if err := open.inventory.SetSettings(ctx, "anchor", "distribution", map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { t.Fatal(err) } var image any for _, r := range composed(t, open, "anchor").Resources { if r["id"] == "postgres.runtime" { image = r["image"] } } if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) } held := heldBy(ctx) if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { t.Fatalf("a builder beside the store is handed the base %q", got) } }