A setting reaches only what declares it, the mesh places its own files, registration refuses a name

Three of novox/hq's group-4 leftovers, one branch.

Issue 173: a module's settings reached every route it contributed, every database it asked for and
every served fact its consumers read — a mail server's site name arrived at the proxy as a route
fact. A setting now overrides a key a contribution or served fact declares and adds none; a file
still merges any key, and a key nothing takes is named as stray instead of dropped silently.

Issue 174: the mesh's own files for a module — its bus credential, its merged config, its bindings —
were placed by the definition under /var/lib/mesh/<module>, 232 host paths in 50 definitions. A
directory may now say `place: "mesh"` and resolves to <root>/mesh/<module>; a directory beneath a
placed one may state its path as `${dir:<id>}/<rest>` and moves with it. The proof test resolves
both catalogues and compares: 48 definitions, no path moved. The controller's own manifest is
converted here; the catalogue in mesh-catalog.

ADR 0155: the installation check moves to registration. `module add` and a build's result both
refuse a definition that names an installation, in the check's words, with the way out; the build
stays recorded.

internal/catalogue/contributes_test.go

@@ -197,6 +197,27 @@ func TestARouteCanBeSetPerMesh(t *testing.T) {
 	}
 }
 
+func TestASettingReachesAContributionOnlyWhereItDeclaresTheKey(t *testing.T) {
+	// novox/hq 04-ISSUES/173: the mail module's site name, set so its environment file could read
+	// it, arrived in every route it contributed. A setting overrides a key the contribution
+	// declares and adds none — the provider reads the contribution as a contract.
+	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
+	out, err := got.Declaration(Rendering{Settings: SettingsBy{
+		"board": {{From: "the mesh", Values: map[string]any{"host": "dashboard", "sitename": "Board"}}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	given := received(t, out)
+	if given[0].Values["host"] != "dashboard" {
+		t.Fatalf("the setting did not override the route's host: %v", given[0].Values)
+	}
+	if _, leaked := given[0].Values["sitename"]; leaked {
+		t.Fatalf("a setting the route never declared reached the proxy: %v", given[0].Values)
+	}
+}
+
 func TestReceivingWhatYouDoNotProvideIsRefused(t *testing.T) {
 	// It would create a file nobody ever writes to, on a machine where nothing asked for it.
 	_, err := ParseManifest([]byte(`{"module":"traefik","version":"1",

internal/catalogue/declaration.go

@@ -1161,7 +1161,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 // module wrote another into its configuration.
 func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers []Layer, what string) (
 	map[string]any, error) {
-	values, err := settle(raw, layers, nil, what)
+	// Overridden, not merged: a setting changes a key the contribution declares and adds none.
+	// The provider reads the contribution as a contract, and a setting made for one of this
+	// module's files is no part of it (novox/hq 04-ISSUES/173).
+	values, err := overridden(raw, layers, what)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", what, err)
 	}

internal/catalogue/dir_into.go

@@ -20,6 +20,12 @@ import (
 // declared with the path as the exception it is, and everything else in the module names it by
 // id — so moving it later is one line, not a search.
 //
+// **The mesh's own files for a module are placed too** (novox/hq issue 174). What the mesh writes
+// *for* a module — its sealed bus credential, its merged configuration, its bindings — is the
+// mesh's plumbing, not the module's data, and sits under `<root>/mesh/<module>`. A directory
+// saying `"place": "mesh"` is that place; the definition names the files beneath it by
+// `${dir:<id>}` and states no path.
+//
 // **Resolved here, not on the machine.** The host receives concrete paths exactly as it always
 // has; nothing new reaches it and it learns no field. Which also means a resolved path changing
 // is a spec change like any other — and the spec comparison must see it (novox/hq issue 126).
@@ -27,6 +33,15 @@ import (
 // defaultDataRoot is where module data lands when a node states no root of its own.
 const defaultDataRoot = "/var/lib"
 
+// The two places a pathless directory may name, beside its own id.
+const (
+	// placeOwn is the assignment's own root, <root>/<module> — to-be 27's one directory per
+	// assignment, which every other placed thing of the module sits beneath.
+	placeOwn = "."
+	// placeMesh is where the mesh keeps what it writes for the module, <root>/mesh/<module>.
+	placeMesh = "mesh"
+)
+
 // dirRef is how a module names one of its placed directories: ${dir:<id>}.
 var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
 
@@ -40,26 +55,46 @@ func dataRoot(with Rendering) string {
 
 // dirsFor is every placed directory of a module, id → the path it resolves to on this node.
 //
-// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module> —
-// to-be 27's one directory per assignment, which every other placed thing sits beneath. At most
-// one makes sense; nothing enforces one, because two ids resolving to one path is a mistake the
-// module's own files make visible immediately.
+// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module>; one
+// saying `"place": "mesh"` is the mesh's directory for the module, <root>/mesh/<module>; one
+// saying neither is <root>/<module>/<id>. At most one of each place makes sense; nothing enforces
+// one, because two ids resolving to one path is a mistake the module's own files make visible
+// immediately.
+//
+// A stated path may itself begin with a placed reference — `${dir:mesh-state}/state` — and is
+// filled after the directories it can name are resolved; one level, because a directory beneath
+// a placed one is the whole of what an adopted layout needs (issue 174's `state` and `out`).
 func dirsFor(m Manifest, with Rendering) map[string]string {
 	dirs := map[string]string{}
+	var beneath []map[string]any
 	for _, r := range m.Resources {
 		if fmt.Sprint(r["type"]) != "directory" {
 			continue
 		}
 		id := fmt.Sprint(r["id"])
 		if path, stated := r["path"].(string); stated && path != "" {
+			if strings.HasPrefix(path, "${dir:") {
+				beneath = append(beneath, r)
+				continue
+			}
 			dirs[id] = strings.TrimRight(path, "/")
 			continue
 		}
-		if place, said := r["place"].(string); said && place == "." {
+		switch place, _ := r["place"].(string); place {
+		case placeOwn:
 			dirs[id] = dataRoot(with) + "/" + m.Module
-			continue
+		case placeMesh:
+			dirs[id] = dataRoot(with) + "/mesh/" + m.Module
+		default:
+			dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
 		}
-		dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
+	}
+	for _, r := range beneath {
+		path := strings.TrimRight(r["path"].(string), "/")
+		// A reference to no directory is left as written and refused where the resource is
+		// placed (dirInto), with the message that names what exists.
+		filled, _ := dirFill(path, dirs, m.Module)
+		dirs[fmt.Sprint(r["id"])] = filled
 	}
 	return dirs
 }
@@ -244,10 +279,11 @@ func (m Manifest) unknownDirRefs() []string {
 				"%s states both path and place on %v — a stated path IS the placement",
 				m.Module, r["id"]))
 		}
-		if place != "." {
+		if place != placeOwn && place != placeMesh {
 			problems = append(problems, fmt.Sprintf(
-				"%s says place %q on %v, and the only place is %q — the assignment's own root",
-				m.Module, place, r["id"], "."))
+				"%s says place %q on %v, and the places are %q — the assignment's own root — and "+
+					"%q — where the mesh keeps what it writes for the module",
+				m.Module, place, r["id"], placeOwn, placeMesh))
 		}
 	}
 	seen := map[string]bool{}

internal/catalogue/dir_into_test.go

@@ -216,8 +216,48 @@ func TestPlaceIsValidatedAtTheManifest(t *testing.T) {
 	wrong := Manifest{Module: "x", Resources: []map[string]any{
 		{"id": "d", "type": "directory", "place": "sub/dir"},
 	}}
-	if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the only place is "."`) {
-		t.Fatalf("a place that is not the root refuses; got %v", got)
+	if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the places are "."`) {
+		t.Fatalf("a place that is neither root refuses; got %v", got)
+	}
+}
+
+func TestTheMeshsDirectoryForAModuleIsAPlace(t *testing.T) {
+	// novox/hq issue 174. What the mesh writes for a module — its bus credential, its bindings —
+	// is the mesh's plumbing under <root>/mesh/<module>, and the definition names it by id.
+	m := Manifest{Module: "umami",
+		Resources: []map[string]any{
+			{"id": "mesh-state", "type": "directory", "place": "mesh"},
+			{"id": "state", "type": "directory", "place": "."},
+			{"id": "server", "type": "container", "image": "x@sha256:aa",
+				"volumes": []any{"${dir:mesh-state}/broker:/run/secrets/broker:ro"}},
+		},
+		OwnSecrets: map[string]string{"broker": "${dir:mesh-state}/broker"},
+		Binds:      map[string]string{"route": "${dir:state}/route.json"},
+	}
+	if got := m.unknownDirRefs(); len(got) != 0 {
+		t.Fatalf("place %q is a place; got %v", "mesh", got)
+	}
+	dirs := dirsFor(m, Rendering{})
+	if dirs["mesh-state"] != "/var/lib/mesh/umami" || dirs["state"] != "/var/lib/umami" {
+		t.Fatalf("the mesh's directory sits beside the module's, not in it; got %v", dirs)
+	}
+	dirs = dirsFor(m, Rendering{DataRoot: "/srv"})
+	if dirs["mesh-state"] != "/srv/mesh/umami" {
+		t.Fatalf("a node's root moves the mesh's files with the module's; got %v", dirs)
+	}
+	placed, err := placedManifest(m, Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if placed.OwnSecrets["broker"] != "/var/lib/mesh/umami/broker" {
+		t.Fatalf("own-secrets are placed under the mesh's directory; got %v", placed.OwnSecrets)
+	}
+	container := shallowCopy(m.Resources[2])
+	if err := dirInto(container, dirsFor(m, Rendering{}), m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if container["volumes"].([]any)[0] != "/var/lib/mesh/umami/broker:/run/secrets/broker:ro" {
+		t.Fatalf("the mount's host side is placed; got %v", container["volumes"])
 	}
 }
 
@@ -250,3 +290,36 @@ func shallowCopy(resource map[string]any) map[string]any {
 	}
 	return copied
 }
+
+func TestADirectoryBeneathAPlacedOneIsPlacedWithIt(t *testing.T) {
+	// An adopted layout keeps a subdirectory the predecessor made under the mesh's directory
+	// (issue 174: a forge's runtime state, a manager's output). Stated as beneath the placed one,
+	// it moves with it — a node's root moves both, and the definition names no host path.
+	m := Manifest{Module: "gitea", Resources: []map[string]any{
+		{"id": "mesh-state", "type": "directory", "place": "mesh"},
+		{"id": "runtime-state", "type": "directory", "path": "${dir:mesh-state}/state"},
+		{"id": "server", "type": "container", "image": "x@sha256:aa",
+			"volumes": []any{"${dir:runtime-state}:/data"}},
+	}}
+	if got := m.unknownDirRefs(); len(got) != 0 {
+		t.Fatalf("a path beneath a placed directory is well formed; got %v", got)
+	}
+	dirs := dirsFor(m, Rendering{DataRoot: "/srv"})
+	if dirs["runtime-state"] != "/srv/mesh/gitea/state" {
+		t.Fatalf("the subdirectory follows the placed one; got %v", dirs)
+	}
+	sub := shallowCopy(m.Resources[1])
+	if err := dirInto(sub, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if sub["path"] != "/srv/mesh/gitea/state" {
+		t.Fatalf("the directory resource itself is resolved; got %v", sub["path"])
+	}
+	container := shallowCopy(m.Resources[2])
+	if err := dirInto(container, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if container["volumes"].([]any)[0] != "/srv/mesh/gitea/state:/data" {
+		t.Fatalf("a reference to the subdirectory resolves whole; got %v", container["volumes"])
+	}
+}

internal/catalogue/placed_paths_test.go

@@ -16,7 +16,9 @@ import (
 //
 // Two checkouts: MESH_CATALOGUE_BEFORE, the catalogue as it was, and MESH_CATALOGUE, as it is now.
 // Every module in both is resolved with the controller's own rule (dirsFor, dirFill) and compared
-// whole — not only the directories, but every string a directory's id was written into.
+// whole — not only the directories, but every string a directory's id was written into. Both
+// sides are resolved, so a manifest converted in two steps (issue 119, then issue 174) is judged
+// against the paths it named, not the text it used to name them with.
 func TestPlacedDirectoriesKeepTheirPaths(t *testing.T) {
 	before, after := os.Getenv("MESH_CATALOGUE_BEFORE"), os.Getenv("MESH_CATALOGUE")
 	if before == "" || after == "" {
@@ -42,7 +44,11 @@ func TestPlacedDirectoriesKeepTheirPaths(t *testing.T) {
 			t.Errorf("%s: %v", module, err)
 			continue
 		}
-		dirs := dirsFor(m, Rendering{})
+		earlier, err := ParseManifest(old)
+		if err != nil {
+			t.Errorf("%s before: %v", module, err)
+			continue
+		}
 		var was, is any
 		if err := json.Unmarshal(old, &was); err != nil {
 			t.Fatal(err)
@@ -50,7 +56,10 @@ func TestPlacedDirectoriesKeepTheirPaths(t *testing.T) {
 		if err := json.Unmarshal(now, &is); err != nil {
 			t.Fatal(err)
 		}
-		resolved := resolvedTree(is, dirs, module, t)
+		// Both sides resolved: the manifest before may itself already place some directories
+		// (issue 119's conversion), and what must not move is the path a machine sees.
+		was = resolvedTree(was, dirsFor(earlier, Rendering{}), module, t)
+		resolved := resolvedTree(is, dirsFor(m, Rendering{}), module, t)
 		if !reflect.DeepEqual(was, resolved) {
 			wasJSON, _ := json.MarshalIndent(was, "", "  ")
 			isJSON, _ := json.MarshalIndent(resolved, "", "  ")

internal/catalogue/settings.go

@@ -85,17 +85,41 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err
 	return out, nil
 }
 
-// Settle lays settings over a module's own values. Exported for what a provider serves, which is
-// settled where the mesh is walked rather than where a node is declared.
+// Settle lays settings over what a provider serves. Exported because a served fact is settled where
+// the mesh is walked rather than where a node is declared.
+//
+// **A setting overrides a served key; it never adds one** (novox/hq 04-ISSUES/173). What a consumer
+// is told is the provider's contract, and a setting made for one of the provider's files — a site
+// name, a public address — is not part of it. Before this, every setting of a module reached every
+// consumer of every provision it served.
 func Settle(base map[string]any, layers []Layer) (map[string]any, error) {
-	return settle(base, layers, nil, "what is served")
+	return overridden(base, layers, "what is served")
+}
+
+// overridden lays settings over a map whose keys are its contract: a contribution, a served fact.
+// Only the keys the map already declares are touched; the rest of a layer is somebody else's
+// business (a file's, another destination's) and is left to reach it there.
+func overridden(base map[string]any, layers []Layer, what string) (map[string]any, error) {
+	kept := make([]Layer, 0, len(layers))
+	for _, layer := range layers {
+		values := map[string]any{}
+		for key, value := range layer.Values {
+			if _, declared := base[key]; declared {
+				values[key] = value
+			}
+		}
+		kept = append(kept, Layer{From: layer.From, Values: values})
+	}
+	return settle(base, kept, nil, what)
 }
 
 // settle lays the layers over a module's own values, in order.
 //
 // Shared by a file's content and a module's contributions, because they are the same act: the
 // module says what it means by default, and somebody says what it means here. A contribution that
-// could not be settled would have to be edited to be reused anywhere else.
+// could not be settled would have to be edited to be reused anywhere else. The two differ in one
+// respect, and the caller decides it: a file takes keys it did not declare (a setting may add to a
+// configuration), a contribution or served fact does not (overridden).
 func settle(base map[string]any, layers []Layer, protected map[string]bool, what string) (
 	map[string]any, error) {
 	merged := deepCopy(base)
@@ -149,23 +173,22 @@ func deepCopy(in map[string]any) map[string]any {
 	return out
 }
 
-// UnusedSettings names settings that reached no file.
+// UnusedSettings names settings that reach nothing.
 //
 // Somebody who sets a key on a module with nothing mergeable, or misspells one, has changed
 // nothing — and would find out by the machine not behaving differently, which is the slowest
 // way there is. This is what makes that visible at the moment they set it.
+//
+// Where a key can land: any mergeable file takes any key; a file asking for `${setting:<key>}`
+// takes that key (ADR 0155); a contribution or a served fact takes a key it declares, and no other
+// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
+// are read by the mesh. A key none of those takes is stray, and is said so rather than dropped.
 func UnusedSettings(m Manifest, layers []Layer) []string {
 	for _, r := range m.Resources {
 		if how, _ := r["merge"].(string); how != "" {
 			return nil
 		}
 	}
-	// A contribution is a destination too. A route's hostname is exactly the kind of thing that
-	// differs between one mesh and the next, and calling it stray would refuse the one setting
-	// most modules that publish anything will have.
-	if len(m.Contributes) > 0 {
-		return nil
-	}
 	// A computed module has no resources here to look at — they are worked out per node, and
 	// whether a setting lands is not knowable until then. Silence rather than a wrong answer:
 	// claiming every setting on the private network is stray would be worse than saying nothing.
@@ -173,12 +196,28 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 		return nil
 	}
 
-	// A key a file's content asks for with ${setting:<key>} is a destination too (ADR 0155).
-	asked := settingKeysUsedBy(m)
+	lands := settingKeysUsedBy(m)
+	for _, values := range m.Contributes {
+		for key := range values {
+			lands[key] = true
+		}
+	}
+	for _, locals := range m.ContributesMany {
+		for _, values := range locals {
+			for key := range values {
+				lands[key] = true
+			}
+		}
+	}
+	for _, served := range m.Serves {
+		for key := range served {
+			lands[key] = true
+		}
+	}
 	var unused []string
 	for _, layer := range layers {
 		for key := range layer.Values {
-			if asked[key] {
+			if lands[key] {
 				continue
 			}
 			// `expose` is a real destination for a module that listens: it overrides a port's
@@ -203,8 +242,9 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 				continue
 			}
 			unused = append(unused, fmt.Sprintf(
-				"%s sets %q, and %s has no file or contribution to merge it into",
-				layer.From, key, m.Module))
+				"%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+
+					"declares no %q in what it contributes or serves",
+				layer.From, key, m.Module, key, key))
 		}
 	}
 	sort.Strings(unused)

internal/catalogue/settings_test.go

@@ -167,11 +167,45 @@ func TestSettingsThatReachNothingAreNamed(t *testing.T) {
 		{"id": "conf", "type": "file", "path": "/etc/thing", "content": "plain"},
 	}}
 	unused := UnusedSettings(m, []Layer{{From: "node", Values: map[string]any{"port": 1}}})
-	if len(unused) != 1 || !strings.Contains(unused[0], "no file or contribution to merge it into") {
+	if len(unused) != 1 || !strings.Contains(unused[0], "no file that merges it") {
 		t.Errorf("settings that reached nothing were not named: %v", unused)
 	}
 }
 
+func TestASettingLandsOnlyWhereSomethingDeclaresIt(t *testing.T) {
+	// novox/hq 04-ISSUES/173. A module that contributes a route and serves a provision takes a
+	// setting for a key either declares, and a setting for a key neither declares is stray — it
+	// would not reach the route or the served fact, so it must be said rather than dropped.
+	m := Manifest{Module: "mail",
+		Contributes: map[string]map[string]any{"reverse-proxy": {"host": "mail", "port": 8080}},
+		Serves:      map[string]map[string]any{"smtp": {"host": "mail", "port": 25}},
+		Resources: []map[string]any{
+			{"id": "env", "type": "file", "path": "/etc/mail.env", "content": "SITE=${setting:sitename}\n"},
+		}}
+	layers := []Layer{{From: "the mesh", Values: map[string]any{
+		"host": "post", "sitename": "Mail", "website": "https://www.example.tld"}}}
+	unused := UnusedSettings(m, layers)
+	if len(unused) != 1 || !strings.Contains(unused[0], `"website"`) {
+		t.Errorf("only website reaches nothing; named: %v", unused)
+	}
+}
+
+func TestASettingOverridesAServedKeyAndAddsNone(t *testing.T) {
+	// What a consumer is told is the provider's contract. A setting made for one of the
+	// provider's files — its site name, its public address — is not part of it.
+	served, err := Settle(map[string]any{"host": "mail", "port": 25},
+		[]Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if served["host"] != "post" {
+		t.Errorf("the setting did not override the served host: %v", served)
+	}
+	if _, leaked := served["sitename"]; leaked {
+		t.Errorf("a setting for a file reached the consumers: %v", served)
+	}
+}
+
 func TestContentThatIsNotJSONIsRefusedWhereSomebodyIsLooking(t *testing.T) {
 	// Rather than on the machine, at apply time, as a file the program cannot read.
 	_, err := ApplySettings(file(`this is not json`), nil)