Merge remote-tracking branch 'origin/main' into feat/900-node-files

cmd/mesh-controller/network_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -244,11 +243,7 @@ func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
 // theResolver is the catalogue's dnsmasq module as it is (internal/beside).
 func theResolver(t *testing.T) catalogue.Manifest {
 	t.Helper()
-	raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "dnsmasq", "module.json"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	m, err := catalogue.ParseManifest(raw)
+	m, err := catalogue.ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), "dnsmasq"))
 	if err != nil {
 		t.Fatalf("dnsmasq does not parse:\n%v", err)
 	}

internal/catalogue/catalogue_check_test.go

@@ -1,7 +1,6 @@
 package catalogue
 
 import (
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -30,11 +29,7 @@ func TestEveryCatalogueManifestParses(t *testing.T) {
 	}
 	named, routed := 0, 0
 	for _, p := range found {
-		raw, err := os.ReadFile(p)
-		if err != nil {
-			t.Fatalf("%s: %v", p, err)
-		}
-		m, err := ParseManifest(raw)
+		m, err := ReadTreeManifest(root, filepath.Dir(p))
 		if err != nil {
 			t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
 			continue
@@ -66,11 +61,7 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
 	}
 	var named []string
 	for _, p := range found {
-		raw, err := os.ReadFile(p)
-		if err != nil {
-			t.Fatalf("%s: %v", p, err)
-		}
-		m, err := ParseManifest(raw)
+		m, err := ReadTreeManifest(root, filepath.Dir(p))
 		if err != nil {
 			t.Errorf("%s: %v", p, err)
 			continue
@@ -95,11 +86,7 @@ func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
 	shelf := Shelf{}
 	granting := 0
 	for _, p := range found {
-		raw, err := os.ReadFile(p)
-		if err != nil {
-			t.Fatalf("%s: %v", p, err)
-		}
-		m, err := ParseManifest(raw)
+		m, err := ReadTreeManifest(root, filepath.Dir(p))
 		if err != nil {
 			continue // TestEveryCatalogueManifestParses says why
 		}
@@ -128,11 +115,7 @@ func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
 	}
 	shelf := Shelf{}
 	for _, p := range found {
-		raw, err := os.ReadFile(p)
-		if err != nil {
-			t.Fatalf("%s: %v", p, err)
-		}
-		m, err := ParseManifest(raw)
+		m, err := ReadTreeManifest(root, filepath.Dir(p))
 		if err != nil {
 			t.Fatalf("%s: %v", p, err)
 		}

internal/catalogue/catrust_manifest_test.go

@@ -1,7 +1,6 @@
 package catalogue
 
 import (
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -19,11 +18,7 @@ import (
 // itself — a plain client trusting an internal name on a machine holding this, and failing on one
 // that does not — is the lab's, and cannot be had here.
 func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
-	raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "ca-trust", "module.json"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	m, err := ParseManifest(raw)
+	m, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), "ca-trust"))
 	if err != nil {
 		t.Fatalf("the trust module does not parse:\n%v", err)
 	}

internal/catalogue/foundation_manifests_test.go

@@ -3,7 +3,6 @@ package catalogue
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -19,11 +18,7 @@ import (
 // filter module loads its table through a unit of its own whose stop deletes only that table.
 func catalogueManifest(t *testing.T, module string) Manifest {
 	t.Helper()
-	raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), module, "module.json"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	m, err := ParseManifest(raw)
+	m, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), module))
 	if err != nil {
 		t.Fatalf("%s does not parse:\n%v", module, err)
 	}
@@ -190,13 +185,14 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
 	if err != nil {
 		t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
 	}
-	r := Resolution{Node: "anchor", Account: "op", Modules: []Manifest{forge, theRuntime(t)}, Needs: []Needed{
-		{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
-			Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
-		{Name: "route", For: "gitea", From: "anchor"},
-		{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
-		{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
-	}}
+	r := Resolution{Node: "anchor", Account: "op", PublicDomain: "example.org",
+		Modules: []Manifest{forge, theRuntime(t)}, Needs: []Needed{
+			{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
+				Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
+			{Name: "route", For: "gitea", From: "anchor"},
+			{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
+			{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
+		}}
 
 	// The number this node was given for the forge — the one the machine it is about to run on
 	// already publishes.
@@ -243,13 +239,14 @@ func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
 	if err != nil {
 		t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
 	}
-	r := Resolution{Node: "anchor", Account: "op", Modules: []Manifest{resolved, theRuntime(t)}, Needs: []Needed{
-		{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
-			Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
-		{Name: "route", For: "gitea", From: "anchor"},
-		{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
-		{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
-	}}
+	r := Resolution{Node: "anchor", Account: "op", PublicDomain: "example.org",
+		Modules: []Manifest{resolved, theRuntime(t)}, Needs: []Needed{
+			{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
+				Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
+			{Name: "route", For: "gitea", From: "anchor"},
+			{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
+			{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
+		}}
 	givenPorts := map[int]int{3000: 3000}
 	for k, v := range given {
 		givenPorts[k] = v
@@ -323,3 +320,35 @@ func forgeBuilt(forge Manifest) []Built {
 	}
 	return out
 }
+
+// catalogueResolved is a catalogue module as the mesh holds it once built: the manifest as written, resolved against
+// what the build produced besides the archive its templates are packed into, which carries the text read back from the
+// module's files/ directory as the real build reads it (novox/hq ADR 0349 §2) — so a module whose file text moved into
+// templates composes with it, and one that did not is resolved as it was.
+func catalogueResolved(t *testing.T, module string, built ...Built) Manifest {
+	t.Helper()
+	dir := filepath.Join(beside.Catalogue(t), module)
+	m, err := ReadTreeManifestWritten(beside.Dir(t, "mesh-catalog"), dir)
+	if err != nil {
+		t.Fatalf("%s does not parse:\n%v", module, err)
+	}
+	if name := TemplatesArtifact(m); name != "" {
+		read, err := ReadTemplates(m, dir)
+		if err != nil {
+			t.Fatalf("%s: %v", module, err)
+		}
+		// By the path inside the archive, as the build reads it back, not by the path in the module's directory.
+		texts := map[string]string{}
+		for path, text := range read {
+			texts[strings.TrimPrefix(path, m.TemplatesDir()+"/")] = text
+		}
+		built = append(built, Built{Name: name, Kind: ArtifactArchive,
+			Reference: ArtifactStoreScheme + module + "/" + name + "/blobs/" + bundleDigest, Digest: bundleDigest,
+			Texts: texts})
+	}
+	held, err := m.Resolve(built)
+	if err != nil {
+		t.Fatalf("%s does not resolve against its own build: %v", module, err)
+	}
+	return held
+}

internal/catalogue/genesis_registry_test.go

@@ -13,13 +13,8 @@ import (
 // agentBuilt is the build agent as the mesh holds it once built: its image pinned.
 func agentBuilt(t *testing.T) Manifest {
 	t.Helper()
-	agent := catalogueManifest(t, "build-agent")
-	built, err := agent.Resolve([]Built{{Name: "server", Kind: ArtifactImage,
-		Reference: "mesh-builder@sha256:" + strings.Repeat("a", 64)}})
-	if err != nil {
-		t.Fatalf("the build agent does not resolve against its own build: %v", err)
-	}
-	return built
+	return catalogueResolved(t, "build-agent", Built{Name: "server", Kind: ArtifactImage,
+		Reference: "mesh-builder@sha256:" + strings.Repeat("a", 64)})
 }
 
 func TestGenesisAssignsTheBuildAgentWithNothingProvidingTheRegistry(t *testing.T) {

internal/catalogue/mounts_test.go

@@ -1,7 +1,6 @@
 package catalogue
 
 import (
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -77,11 +76,7 @@ func TestEveryCatalogueManifestDeclaresWhatItMounts(t *testing.T) {
 		t.Fatal("no manifests in the catalogue, so this proved nothing")
 	}
 	for _, file := range files {
-		raw, err := os.ReadFile(file)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if _, err := ParseManifest(raw); err != nil {
+		if _, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Dir(file)); err != nil {
 			t.Errorf("%s: %v", filepath.Base(filepath.Dir(file)), err)
 		}
 	}

internal/catalogue/predefined.go

@@ -234,3 +234,30 @@ func ReplacePublished(list PredefinedList) (was PredefinedList) {
 	was, publishedPredefined.list = publishedPredefined.list, list
 	return was
 }
+
+// ReadTreeManifestWritten is the manifest in a module's directory of a repository tree as written, still naming its
+// templates, judged against the list of predefined keys at the tree's root as the merge gate judges it
+// (cmd/mesh-controller). What a build resolves, given the text it reads back from the archive it packs the templates
+// into (Built.Texts, ReadTemplates).
+func ReadTreeManifestWritten(root, dir string) (Manifest, error) {
+	list, err := ReadPredefined(root)
+	if err != nil {
+		return Manifest{}, err
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
+	if err != nil {
+		return Manifest{}, err
+	}
+	return ParseManifestWith(raw, list)
+}
+
+// ReadTreeManifest is ReadTreeManifestWritten with each template's text read from the module's directory as the
+// merge gate reads it (WithTemplatesFrom). For a check that composes a tree's manifests, so what it judges is what
+// the real check judges.
+func ReadTreeManifest(root, dir string) (Manifest, error) {
+	m, err := ReadTreeManifestWritten(root, dir)
+	if err != nil {
+		return Manifest{}, err
+	}
+	return WithTemplatesFrom(m, dir)
+}

internal/catalogue/reads_test.go

@@ -74,7 +74,8 @@ func TestTheHolderOfNodeReadsIsToldWhereItsNodeKeepsASecret(t *testing.T) {
 	desk.RunsAs, desk.SecretsOwner = "desk", "desk"
 	desk.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/desk/broker"}}
 	// The vault as the catalogue declares it: its directories have no path of their own, and the node places them.
-	vault, err := placedManifest(catalogueManifest(t, "mesh-vault"), with)
+	vault, err := placedManifest(catalogueResolved(t, "mesh-vault", Built{Name: "code", Kind: ArtifactBundle,
+		Reference: ArtifactStoreScheme + "mesh-vault/code/blobs/" + bundleDigest, Digest: bundleDigest}), with)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/catalogue/seat_data_test.go

@@ -157,22 +157,39 @@ func TestAPieceTheHolderCannotRenderIsLeftOutAndNamedNotTheMachine(t *testing.T)
 	}
 }
 
-func TestTheCataloguesBarRendersEveryExampleOfTheShape(t *testing.T) {
+// A moved bar is rendered by the provisioner's client, not by the controller, so the rule that its template renders
+// every example of the shape is no longer judged here (issue 915, which also covers placementProblems not reading the
+// collected form). What the controller still owns is that the bar's files place the blocks at all: a moved bar's
+// carry the collected placement, and one that stopped placing them fails here rather than being skipped (issue 432).
+func TestTheCataloguesBarPlacesBlocksAndItsShapeCoversEveryShow(t *testing.T) {
 	holder := catalogueManifest(t, "i3status-rust")
 	s, _ := SeatNamed(BarSeat)
-	if !placesKind(holder, s, BarKindBlock) {
-		// It places them since the catalogue's bar took the seat; a skip here hid a bar that stopped
-		// (novox/hq issue 432).
-		t.Fatal("the catalogue's bar places no blocks, so none of the shape's examples would render")
-	}
-	tmpl, err := holderTemplate(holder, s, BarKindBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, ex := range s.Receives[0].Shape.Examples {
-		out, err := renderPiece(tmpl, "example", "a-machine", ex)
-		if err != nil || !strings.Contains(out, "[[block]]") {
-			t.Errorf("the catalogue's bar renders %v as %q: %v", ex, out, err)
+	if holder.Moved {
+		placed := "${collected:contribution:" + BarSeat + ":" + BarKindBlock + ":"
+		found := false
+		for _, res := range holder.Resources {
+			if content, _ := res["content"].(string); strings.Contains(content, placed) {
+				found = true
+			}
+		}
+		if !found {
+			t.Fatalf("the catalogue's moved bar places no blocks: none of its files carries %s…}", placed)
+		}
+	} else {
+		if !placesKind(holder, s, BarKindBlock) {
+			// It places them since the catalogue's bar took the seat; a skip here hid a bar that stopped
+			// (novox/hq issue 432).
+			t.Fatal("the catalogue's bar places no blocks, so none of the shape's examples would render")
+		}
+		tmpl, err := holderTemplate(holder, s, BarKindBlock)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, ex := range s.Receives[0].Shape.Examples {
+			out, err := renderPiece(tmpl, "example", "a-machine", ex)
+			if err != nil || !strings.Contains(out, "[[block]]") {
+				t.Errorf("the catalogue's bar renders %v as %q: %v", ex, out, err)
+			}
 		}
 	}
 	for shows := range barShows {