Merge pull request 'A directory the mesh places: ${dir:<id>} and the pathless directory resource' (#69) from feat/a-directory-the-mesh-places into main

internal/catalogue/declaration.go

@@ -168,6 +168,13 @@ type Rendering struct {
 	// with an address — before references were kept without one — from an image a module runs
 	// straight from a public registry.
 	Built map[string]bool
+
+	// DataRoot is where this node keeps the directories the mesh places for its modules
+	// (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to
+	// <DataRoot>/<module>/<id>, and ${dir:<id>} names that place from the module's own files,
+	// mounts and environment. A node setting fixed at installation; empty means the default,
+	// /var/lib — see dir_into.go.
+	DataRoot string
 }
 
 // machinePort is where a module's port lives on this machine, or the port itself when the mesh has
@@ -519,6 +526,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
 		}
 		// And what its bindings say, for the half of a connection that is not secret.
 		known := knownFor(m, r.Needs, r.Node)
+		// And where this node places the directories the module declared without a path
+		// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
+		dirs := dirsFor(m, with)
 		// And the machine underneath, which no binding of its own can tell it.
 		thisMachine := machineFacts(r, with.Names)
 
@@ -541,6 +551,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
 			// Said in the catalogue, not on the machine: the host parses strictly and knows no
 			// such field, and the reason is for a reader of the manifest.
 			delete(copied, SecretsInEnvironment)
+			// **Placed before anything reads a path.** A pathless directory receives the path
+			// this node resolves for it, and every ${dir:…} — in paths, mounts, content and
+			// environment — becomes that path, so what follows sees only concrete places
+			// (novox/hq ADR 0112). The host receives paths exactly as it always has.
+			if err := dirInto(copied, dirs, m.Module); err != nil {
+				return nil, err
+			}
 			// **After settings, and that is the whole reason it is here.** A module's file
 			// content is where a setting lands, so a placeholder may only exist once the setting
 			// has been put in — filling secrets first would look at content that is not yet what

internal/catalogue/dir_into.go

@@ -0,0 +1,211 @@
+package catalogue
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// A directory the mesh places (novox/hq ADR 0112, to-be 27, issue 119).
+//
+// **A module definition names no host path.** A directory resource may omit `path`; the mesh
+// resolves where it lands when the declaration is composed — `<root>/<module>/<id>`, the root a
+// node's own setting with /var/lib as the default. From then on the module's own files, mounts
+// and environment name the place as `${dir:<id>}`, the same shape as `${bound:…}` and
+// `${secret:…}`: a fact the module asks for by name and never states.
+//
+// **A directory that states a path keeps it, and still answers `${dir:<id>}`.** That is the
+// placement for an adopted machine: data that must sit where the predecessor already put it is
+// declared with the path as the exception it is, and everything else in the module names it by
+// id — so moving it later is one line, not a search.
+//
+// **Resolved here, not on the machine.** The host receives concrete paths exactly as it always
+// has; nothing new reaches it and it learns no field. Which also means a resolved path changing
+// is a spec change like any other — and the spec comparison must see it (novox/hq issue 126).
+
+// defaultDataRoot is where module data lands when a node states no root of its own.
+const defaultDataRoot = "/var/lib"
+
+// dirRef is how a module names one of its placed directories: ${dir:<id>}.
+var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
+
+// dataRoot is the root this node keeps placed directories under.
+func dataRoot(with Rendering) string {
+	if root := strings.TrimRight(strings.TrimSpace(with.DataRoot), "/"); root != "" {
+		return root
+	}
+	return defaultDataRoot
+}
+
+// dirsFor is every placed directory of a module, id → the path it resolves to on this node.
+func dirsFor(m Manifest, with Rendering) map[string]string {
+	dirs := map[string]string{}
+	for _, r := range m.Resources {
+		if fmt.Sprint(r["type"]) != "directory" {
+			continue
+		}
+		id := fmt.Sprint(r["id"])
+		if path, stated := r["path"].(string); stated && path != "" {
+			dirs[id] = strings.TrimRight(path, "/")
+			continue
+		}
+		dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
+	}
+	return dirs
+}
+
+// dirInto places a resource: a pathless directory is given the path the mesh resolved for it,
+// and every ${dir:…} the resource carries — in its path, its content, its mounts, its
+// environment and its env-files — becomes that path.
+//
+// A reference naming no directory of this module is refused. Left as written, the literal
+// `${dir:x}` would reach the machine as a path, and the runtime would create and mount a
+// directory called `${dir:x}` — real, wrong, and named after the mistake.
+func dirInto(resource map[string]any, dirs map[string]string, module string) error {
+	fill := func(s string) (string, error) {
+		var missing error
+		out := dirRef.ReplaceAllStringFunc(s, func(ref string) string {
+			id := dirRef.FindStringSubmatch(ref)[1]
+			path, has := dirs[id]
+			if !has {
+				missing = fmt.Errorf(
+					"%s says ${dir:%s}, and %s declares no directory %q. It declares %s",
+					module, id, module, id, orNothing(namesOfDirs(dirs)))
+				return ref
+			}
+			return path
+		})
+		return out, missing
+	}
+
+	if fmt.Sprint(resource["type"]) == "directory" {
+		id := fmt.Sprint(resource["id"])
+		if path, stated := resource["path"].(string); !stated || path == "" {
+			resource["path"] = dirs[id]
+		}
+	}
+
+	var err error
+	if path, ok := resource["path"].(string); ok {
+		if resource["path"], err = fill(path); err != nil {
+			return err
+		}
+	}
+	if content, ok := resource["content"].(string); ok {
+		if resource["content"], err = fill(content); err != nil {
+			return err
+		}
+	}
+	// Nested values are rebuilt, never written into: the resource is a shallow copy of the
+	// manifest's own map, and the manifest is composed once per node — a fill written in place
+	// would leave the first node's paths inside every later composition.
+	if volumes, ok := resource["volumes"].([]any); ok {
+		filled := make([]any, len(volumes))
+		for i, v := range volumes {
+			filled[i] = v
+			if mount, ok := v.(string); ok {
+				if filled[i], err = fill(mount); err != nil {
+					return err
+				}
+			}
+		}
+		resource["volumes"] = filled
+	}
+	if env, ok := resource["env"].(map[string]any); ok {
+		filled := make(map[string]any, len(env))
+		for key, v := range env {
+			filled[key] = v
+			if value, ok := v.(string); ok {
+				if filled[key], err = fill(value); err != nil {
+					return err
+				}
+			}
+		}
+		resource["env"] = filled
+	}
+	if files, ok := resource["env-file"].([]any); ok {
+		filled := make([]any, len(files))
+		for i, v := range files {
+			filled[i] = v
+			if path, ok := v.(string); ok {
+				if filled[i], err = fill(path); err != nil {
+					return err
+				}
+			}
+		}
+		resource["env-file"] = filled
+	}
+	return nil
+}
+
+// unknownDirRefs is every ${dir:…} in the definition that names no directory the definition
+// declares — refused where the author is, not at composition on some later day (the same
+// near-versus-far reasoning as the host's strict parse).
+func (m Manifest) unknownDirRefs() []string {
+	declared := map[string]bool{}
+	for _, r := range m.Resources {
+		if fmt.Sprint(r["type"]) == "directory" {
+			declared[fmt.Sprint(r["id"])] = true
+		}
+	}
+	referenced := func(s string) []string {
+		var ids []string
+		for _, match := range dirRef.FindAllStringSubmatch(s, -1) {
+			ids = append(ids, match[1])
+		}
+		return ids
+	}
+	var problems []string
+	seen := map[string]bool{}
+	refuse := func(id string, where any) {
+		if declared[id] || seen[id] {
+			return
+		}
+		seen[id] = true
+		problems = append(problems, fmt.Sprintf(
+			"%s says ${dir:%s} in %v, and declares no directory %q — a reference the mesh "+
+				"cannot place would reach the machine as a literal path",
+			m.Module, id, where, id))
+	}
+	for _, r := range m.Resources {
+		for _, field := range []string{"path", "content"} {
+			if s, ok := r[field].(string); ok {
+				for _, id := range referenced(s) {
+					refuse(id, r["id"])
+				}
+			}
+		}
+		for _, field := range []string{"volumes", "env-file"} {
+			if list, ok := r[field].([]any); ok {
+				for _, v := range list {
+					if s, ok := v.(string); ok {
+						for _, id := range referenced(s) {
+							refuse(id, r["id"])
+						}
+					}
+				}
+			}
+		}
+		if env, ok := r["env"].(map[string]any); ok {
+			for _, v := range env {
+				if s, ok := v.(string); ok {
+					for _, id := range referenced(s) {
+						refuse(id, r["id"])
+					}
+				}
+			}
+		}
+	}
+	sort.Strings(problems)
+	return problems
+}
+
+func namesOfDirs(dirs map[string]string) []string {
+	var names []string
+	for id := range dirs {
+		names = append(names, fmt.Sprintf("%q", id))
+	}
+	sort.Strings(names)
+	return names
+}

internal/catalogue/dir_into_test.go

@@ -0,0 +1,141 @@
+package catalogue
+
+// A directory the mesh places (novox/hq ADR 0112). These tests pin the contract: a pathless
+// directory resolves under the node's root, ${dir:…} names it from every field a host path can
+// live in, a stated path is the adopted-data placement and wins, an unknown reference refuses at
+// the manifest, and filling for one node never leaks into the next composition.
+
+import (
+	"strings"
+	"testing"
+)
+
+func placedModule() Manifest {
+	return Manifest{
+		Module: "photos",
+		Resources: []map[string]any{
+			{"id": "data", "type": "directory", "mode": "0700"},
+			{"id": "server-env", "type": "file", "path": "${dir:data}/server.env",
+				"content": "STORE=${dir:data}/objects\n"},
+			{"id": "server", "type": "container", "name": "photos-server",
+				"volumes":  []any{"${dir:data}:/data"},
+				"env":      map[string]any{"DATA": "${dir:data}/objects"},
+				"env-file": []any{"${dir:data}/server.env"}},
+		},
+	}
+}
+
+func TestAPathlessDirectoryResolvesUnderTheNodesRoot(t *testing.T) {
+	m := placedModule()
+	dirs := dirsFor(m, Rendering{})
+	if dirs["data"] != "/var/lib/photos/data" {
+		t.Fatalf("the default root is /var/lib and the shape is <root>/<module>/<id>; got %q", dirs["data"])
+	}
+	dirs = dirsFor(m, Rendering{DataRoot: "/tank/nox/"})
+	if dirs["data"] != "/tank/nox/photos/data" {
+		t.Fatalf("a node's own root is honoured, trailing slash and all; got %q", dirs["data"])
+	}
+}
+
+func TestDirReferencesBecomeThePlaceInEveryField(t *testing.T) {
+	m := placedModule()
+	dirs := dirsFor(m, Rendering{})
+
+	directory := shallowCopy(m.Resources[0])
+	if err := dirInto(directory, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if directory["path"] != "/var/lib/photos/data" {
+		t.Fatalf("a pathless directory receives its resolved path; got %v", directory["path"])
+	}
+
+	file := shallowCopy(m.Resources[1])
+	if err := dirInto(file, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if file["path"] != "/var/lib/photos/data/server.env" {
+		t.Fatalf("a file's path names the place; got %v", file["path"])
+	}
+	if file["content"] != "STORE=/var/lib/photos/data/objects\n" {
+		t.Fatalf("a file's content names the place; got %v", file["content"])
+	}
+
+	container := shallowCopy(m.Resources[2])
+	if err := dirInto(container, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if container["volumes"].([]any)[0] != "/var/lib/photos/data:/data" {
+		t.Fatalf("a mount names the place; got %v", container["volumes"])
+	}
+	if container["env"].(map[string]any)["DATA"] != "/var/lib/photos/data/objects" {
+		t.Fatalf("an environment value names the place; got %v", container["env"])
+	}
+	if container["env-file"].([]any)[0] != "/var/lib/photos/data/server.env" {
+		t.Fatalf("an env-file names the place; got %v", container["env-file"])
+	}
+}
+
+func TestAStatedPathIsThePlacementAndStillAnswersByName(t *testing.T) {
+	m := placedModule()
+	// The adopted-machine case: data that must sit where the predecessor already put it.
+	m.Resources[0]["path"] = "/services/mssql/data/"
+	dirs := dirsFor(m, Rendering{})
+	if dirs["data"] != "/services/mssql/data" {
+		t.Fatalf("a stated path wins over the root, trimmed; got %q", dirs["data"])
+	}
+	container := shallowCopy(m.Resources[2])
+	if err := dirInto(container, dirs, m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if container["volumes"].([]any)[0] != "/services/mssql/data:/data" {
+		t.Fatalf("references follow the placement; got %v", container["volumes"])
+	}
+}
+
+func TestFillingForOneNodeLeaksIntoNoOther(t *testing.T) {
+	m := placedModule()
+	first := shallowCopy(m.Resources[2])
+	if err := dirInto(first, dirsFor(m, Rendering{DataRoot: "/first"}), m.Module); err != nil {
+		t.Fatal(err)
+	}
+	second := shallowCopy(m.Resources[2])
+	if err := dirInto(second, dirsFor(m, Rendering{DataRoot: "/second"}), m.Module); err != nil {
+		t.Fatal(err)
+	}
+	if got := second["volumes"].([]any)[0]; got != "/second/photos/data:/data" {
+		t.Fatalf("the second composition must see the manifest, not the first fill; got %v", got)
+	}
+	if m.Resources[2]["volumes"].([]any)[0] != "${dir:data}:/data" {
+		t.Fatalf("the manifest itself stays a template; got %v", m.Resources[2]["volumes"])
+	}
+}
+
+func TestAReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) {
+	m := placedModule()
+	m.Resources[2]["volumes"] = []any{"${dir:date}:/data"} // a typo, the likely shape
+	problems := m.unknownDirRefs()
+	if len(problems) != 1 || !strings.Contains(problems[0], `${dir:date}`) {
+		t.Fatalf("a reference naming no directory is a manifest problem; got %v", problems)
+	}
+	if got := placedModule().unknownDirRefs(); len(got) != 0 {
+		t.Fatalf("a correct definition has none; got %v", got)
+	}
+}
+
+func TestAReferenceToNoDirectoryRefusesAtCompositionToo(t *testing.T) {
+	m := placedModule()
+	container := shallowCopy(m.Resources[2])
+	container["env"] = map[string]any{"DATA": "${dir:date}"}
+	err := dirInto(container, dirsFor(m, Rendering{}), m.Module)
+	if err == nil || !strings.Contains(err.Error(), `"date"`) || !strings.Contains(err.Error(), `"data"`) {
+		t.Fatalf("the refusal names the mistake and what exists; got %v", err)
+	}
+}
+
+func shallowCopy(resource map[string]any) map[string]any {
+	copied := map[string]any{}
+	for k, v := range resource {
+		copied[k] = v
+	}
+	return copied
+}

internal/catalogue/manifest.go

@@ -1324,6 +1324,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
 	// Checked here rather than on the machine because the machine cannot tell the difference: by
 	// the time it sees the mount it is being asked to create the directory, which it can do.
 	problems = append(problems, m.undeclaredMounts()...)
+	problems = append(problems, m.unknownDirRefs()...)
 
 	for i, r := range m.Resources {
 		id, _ := r["id"].(string)