From d2cbc9dbdc7ab3369becd6883a1024a48921b149 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 17:51:54 +0200 Subject: [PATCH] A directory the mesh places: ${dir:} and the pathless directory resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first executable slice of ADR 0112 / to-be 27, sized to what the operator settled tonight: a module definition names no host path for its own data. A directory resource may omit path; composition resolves it to //, the root a node's setting on Rendering with /var/lib as the default — which reproduces exactly the layout novox converged to by hand. ${dir:} names the place from a resource's path, content, mounts, environment and env-files, the same shape as ${bound:…}. A directory that states a path keeps it and still answers by name — that is the adopted-data placement, mssql its live case. Resolved in the controller at composition, so the wire format and the host change not at all; a reference naming no directory refuses at the manifest and again at composition; nested fills are rebuilt, never written into the manifest's own maps, because one manifest composes for many nodes. --- internal/catalogue/declaration.go | 17 +++ internal/catalogue/dir_into.go | 211 ++++++++++++++++++++++++++++ internal/catalogue/dir_into_test.go | 141 +++++++++++++++++++ internal/catalogue/manifest.go | 1 + 4 files changed, 370 insertions(+) create mode 100644 internal/catalogue/dir_into.go create mode 100644 internal/catalogue/dir_into_test.go diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 9fbee3a..79dc773 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -168,6 +168,13 @@ type Rendering struct { // with an address — before references were kept without one — from an image a module runs // straight from a public registry. Built map[string]bool + + // DataRoot is where this node keeps the directories the mesh places for its modules + // (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to + // //, and ${dir:} names that place from the module's own files, + // mounts and environment. A node setting fixed at installation; empty means the default, + // /var/lib — see dir_into.go. + DataRoot string } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has @@ -519,6 +526,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri } // And what its bindings say, for the half of a connection that is not secret. known := knownFor(m, r.Needs, r.Node) + // And where this node places the directories the module declared without a path + // (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource. + dirs := dirsFor(m, with) // And the machine underneath, which no binding of its own can tell it. thisMachine := machineFacts(r, with.Names) @@ -541,6 +551,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri // Said in the catalogue, not on the machine: the host parses strictly and knows no // such field, and the reason is for a reader of the manifest. delete(copied, SecretsInEnvironment) + // **Placed before anything reads a path.** A pathless directory receives the path + // this node resolves for it, and every ${dir:…} — in paths, mounts, content and + // environment — becomes that path, so what follows sees only concrete places + // (novox/hq ADR 0112). The host receives paths exactly as it always has. + if err := dirInto(copied, dirs, m.Module); err != nil { + return nil, err + } // **After settings, and that is the whole reason it is here.** A module's file // content is where a setting lands, so a placeholder may only exist once the setting // has been put in — filling secrets first would look at content that is not yet what diff --git a/internal/catalogue/dir_into.go b/internal/catalogue/dir_into.go new file mode 100644 index 0000000..5802e43 --- /dev/null +++ b/internal/catalogue/dir_into.go @@ -0,0 +1,211 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// A directory the mesh places (novox/hq ADR 0112, to-be 27, issue 119). +// +// **A module definition names no host path.** A directory resource may omit `path`; the mesh +// resolves where it lands when the declaration is composed — `//`, the root a +// node's own setting with /var/lib as the default. From then on the module's own files, mounts +// and environment name the place as `${dir:}`, the same shape as `${bound:…}` and +// `${secret:…}`: a fact the module asks for by name and never states. +// +// **A directory that states a path keeps it, and still answers `${dir:}`.** That is the +// placement for an adopted machine: data that must sit where the predecessor already put it is +// declared with the path as the exception it is, and everything else in the module names it by +// id — so moving it later is one line, not a search. +// +// **Resolved here, not on the machine.** The host receives concrete paths exactly as it always +// has; nothing new reaches it and it learns no field. Which also means a resolved path changing +// is a spec change like any other — and the spec comparison must see it (novox/hq issue 126). + +// defaultDataRoot is where module data lands when a node states no root of its own. +const defaultDataRoot = "/var/lib" + +// dirRef is how a module names one of its placed directories: ${dir:}. +var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`) + +// dataRoot is the root this node keeps placed directories under. +func dataRoot(with Rendering) string { + if root := strings.TrimRight(strings.TrimSpace(with.DataRoot), "/"); root != "" { + return root + } + return defaultDataRoot +} + +// dirsFor is every placed directory of a module, id → the path it resolves to on this node. +func dirsFor(m Manifest, with Rendering) map[string]string { + dirs := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + id := fmt.Sprint(r["id"]) + if path, stated := r["path"].(string); stated && path != "" { + dirs[id] = strings.TrimRight(path, "/") + continue + } + dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id + } + return dirs +} + +// dirInto places a resource: a pathless directory is given the path the mesh resolved for it, +// and every ${dir:…} the resource carries — in its path, its content, its mounts, its +// environment and its env-files — becomes that path. +// +// A reference naming no directory of this module is refused. Left as written, the literal +// `${dir:x}` would reach the machine as a path, and the runtime would create and mount a +// directory called `${dir:x}` — real, wrong, and named after the mistake. +func dirInto(resource map[string]any, dirs map[string]string, module string) error { + fill := func(s string) (string, error) { + var missing error + out := dirRef.ReplaceAllStringFunc(s, func(ref string) string { + id := dirRef.FindStringSubmatch(ref)[1] + path, has := dirs[id] + if !has { + missing = fmt.Errorf( + "%s says ${dir:%s}, and %s declares no directory %q. It declares %s", + module, id, module, id, orNothing(namesOfDirs(dirs))) + return ref + } + return path + }) + return out, missing + } + + if fmt.Sprint(resource["type"]) == "directory" { + id := fmt.Sprint(resource["id"]) + if path, stated := resource["path"].(string); !stated || path == "" { + resource["path"] = dirs[id] + } + } + + var err error + if path, ok := resource["path"].(string); ok { + if resource["path"], err = fill(path); err != nil { + return err + } + } + if content, ok := resource["content"].(string); ok { + if resource["content"], err = fill(content); err != nil { + return err + } + } + // Nested values are rebuilt, never written into: the resource is a shallow copy of the + // manifest's own map, and the manifest is composed once per node — a fill written in place + // would leave the first node's paths inside every later composition. + if volumes, ok := resource["volumes"].([]any); ok { + filled := make([]any, len(volumes)) + for i, v := range volumes { + filled[i] = v + if mount, ok := v.(string); ok { + if filled[i], err = fill(mount); err != nil { + return err + } + } + } + resource["volumes"] = filled + } + if env, ok := resource["env"].(map[string]any); ok { + filled := make(map[string]any, len(env)) + for key, v := range env { + filled[key] = v + if value, ok := v.(string); ok { + if filled[key], err = fill(value); err != nil { + return err + } + } + } + resource["env"] = filled + } + if files, ok := resource["env-file"].([]any); ok { + filled := make([]any, len(files)) + for i, v := range files { + filled[i] = v + if path, ok := v.(string); ok { + if filled[i], err = fill(path); err != nil { + return err + } + } + } + resource["env-file"] = filled + } + return nil +} + +// unknownDirRefs is every ${dir:…} in the definition that names no directory the definition +// declares — refused where the author is, not at composition on some later day (the same +// near-versus-far reasoning as the host's strict parse). +func (m Manifest) unknownDirRefs() []string { + declared := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + declared[fmt.Sprint(r["id"])] = true + } + } + referenced := func(s string) []string { + var ids []string + for _, match := range dirRef.FindAllStringSubmatch(s, -1) { + ids = append(ids, match[1]) + } + return ids + } + var problems []string + seen := map[string]bool{} + refuse := func(id string, where any) { + if declared[id] || seen[id] { + return + } + seen[id] = true + problems = append(problems, fmt.Sprintf( + "%s says ${dir:%s} in %v, and declares no directory %q — a reference the mesh "+ + "cannot place would reach the machine as a literal path", + m.Module, id, where, id)) + } + for _, r := range m.Resources { + for _, field := range []string{"path", "content"} { + if s, ok := r[field].(string); ok { + for _, id := range referenced(s) { + refuse(id, r["id"]) + } + } + } + for _, field := range []string{"volumes", "env-file"} { + if list, ok := r[field].([]any); ok { + for _, v := range list { + if s, ok := v.(string); ok { + for _, id := range referenced(s) { + refuse(id, r["id"]) + } + } + } + } + } + if env, ok := r["env"].(map[string]any); ok { + for _, v := range env { + if s, ok := v.(string); ok { + for _, id := range referenced(s) { + refuse(id, r["id"]) + } + } + } + } + } + sort.Strings(problems) + return problems +} + +func namesOfDirs(dirs map[string]string) []string { + var names []string + for id := range dirs { + names = append(names, fmt.Sprintf("%q", id)) + } + sort.Strings(names) + return names +} diff --git a/internal/catalogue/dir_into_test.go b/internal/catalogue/dir_into_test.go new file mode 100644 index 0000000..661ae63 --- /dev/null +++ b/internal/catalogue/dir_into_test.go @@ -0,0 +1,141 @@ +package catalogue + +// A directory the mesh places (novox/hq ADR 0112). These tests pin the contract: a pathless +// directory resolves under the node's root, ${dir:…} names it from every field a host path can +// live in, a stated path is the adopted-data placement and wins, an unknown reference refuses at +// the manifest, and filling for one node never leaks into the next composition. + +import ( + "strings" + "testing" +) + +func placedModule() Manifest { + return Manifest{ + Module: "photos", + Resources: []map[string]any{ + {"id": "data", "type": "directory", "mode": "0700"}, + {"id": "server-env", "type": "file", "path": "${dir:data}/server.env", + "content": "STORE=${dir:data}/objects\n"}, + {"id": "server", "type": "container", "name": "photos-server", + "volumes": []any{"${dir:data}:/data"}, + "env": map[string]any{"DATA": "${dir:data}/objects"}, + "env-file": []any{"${dir:data}/server.env"}}, + }, + } +} + +func TestAPathlessDirectoryResolvesUnderTheNodesRoot(t *testing.T) { + m := placedModule() + dirs := dirsFor(m, Rendering{}) + if dirs["data"] != "/var/lib/photos/data" { + t.Fatalf("the default root is /var/lib and the shape is //; got %q", dirs["data"]) + } + dirs = dirsFor(m, Rendering{DataRoot: "/tank/nox/"}) + if dirs["data"] != "/tank/nox/photos/data" { + t.Fatalf("a node's own root is honoured, trailing slash and all; got %q", dirs["data"]) + } +} + +func TestDirReferencesBecomeThePlaceInEveryField(t *testing.T) { + m := placedModule() + dirs := dirsFor(m, Rendering{}) + + directory := shallowCopy(m.Resources[0]) + if err := dirInto(directory, dirs, m.Module); err != nil { + t.Fatal(err) + } + if directory["path"] != "/var/lib/photos/data" { + t.Fatalf("a pathless directory receives its resolved path; got %v", directory["path"]) + } + + file := shallowCopy(m.Resources[1]) + if err := dirInto(file, dirs, m.Module); err != nil { + t.Fatal(err) + } + if file["path"] != "/var/lib/photos/data/server.env" { + t.Fatalf("a file's path names the place; got %v", file["path"]) + } + if file["content"] != "STORE=/var/lib/photos/data/objects\n" { + t.Fatalf("a file's content names the place; got %v", file["content"]) + } + + container := shallowCopy(m.Resources[2]) + if err := dirInto(container, dirs, m.Module); err != nil { + t.Fatal(err) + } + if container["volumes"].([]any)[0] != "/var/lib/photos/data:/data" { + t.Fatalf("a mount names the place; got %v", container["volumes"]) + } + if container["env"].(map[string]any)["DATA"] != "/var/lib/photos/data/objects" { + t.Fatalf("an environment value names the place; got %v", container["env"]) + } + if container["env-file"].([]any)[0] != "/var/lib/photos/data/server.env" { + t.Fatalf("an env-file names the place; got %v", container["env-file"]) + } +} + +func TestAStatedPathIsThePlacementAndStillAnswersByName(t *testing.T) { + m := placedModule() + // The adopted-machine case: data that must sit where the predecessor already put it. + m.Resources[0]["path"] = "/services/mssql/data/" + dirs := dirsFor(m, Rendering{}) + if dirs["data"] != "/services/mssql/data" { + t.Fatalf("a stated path wins over the root, trimmed; got %q", dirs["data"]) + } + container := shallowCopy(m.Resources[2]) + if err := dirInto(container, dirs, m.Module); err != nil { + t.Fatal(err) + } + if container["volumes"].([]any)[0] != "/services/mssql/data:/data" { + t.Fatalf("references follow the placement; got %v", container["volumes"]) + } +} + +func TestFillingForOneNodeLeaksIntoNoOther(t *testing.T) { + m := placedModule() + first := shallowCopy(m.Resources[2]) + if err := dirInto(first, dirsFor(m, Rendering{DataRoot: "/first"}), m.Module); err != nil { + t.Fatal(err) + } + second := shallowCopy(m.Resources[2]) + if err := dirInto(second, dirsFor(m, Rendering{DataRoot: "/second"}), m.Module); err != nil { + t.Fatal(err) + } + if got := second["volumes"].([]any)[0]; got != "/second/photos/data:/data" { + t.Fatalf("the second composition must see the manifest, not the first fill; got %v", got) + } + if m.Resources[2]["volumes"].([]any)[0] != "${dir:data}:/data" { + t.Fatalf("the manifest itself stays a template; got %v", m.Resources[2]["volumes"]) + } +} + +func TestAReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) { + m := placedModule() + m.Resources[2]["volumes"] = []any{"${dir:date}:/data"} // a typo, the likely shape + problems := m.unknownDirRefs() + if len(problems) != 1 || !strings.Contains(problems[0], `${dir:date}`) { + t.Fatalf("a reference naming no directory is a manifest problem; got %v", problems) + } + if got := placedModule().unknownDirRefs(); len(got) != 0 { + t.Fatalf("a correct definition has none; got %v", got) + } +} + +func TestAReferenceToNoDirectoryRefusesAtCompositionToo(t *testing.T) { + m := placedModule() + container := shallowCopy(m.Resources[2]) + container["env"] = map[string]any{"DATA": "${dir:date}"} + err := dirInto(container, dirsFor(m, Rendering{}), m.Module) + if err == nil || !strings.Contains(err.Error(), `"date"`) || !strings.Contains(err.Error(), `"data"`) { + t.Fatalf("the refusal names the mistake and what exists; got %v", err) + } +} + +func shallowCopy(resource map[string]any) map[string]any { + copied := map[string]any{} + for k, v := range resource { + copied[k] = v + } + return copied +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index ebe2fae..cf56977 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -1324,6 +1324,7 @@ func ParseManifest(raw []byte) (Manifest, error) { // Checked here rather than on the machine because the machine cannot tell the difference: by // the time it sees the mount it is being asked to create the directory, which it can do. problems = append(problems, m.undeclaredMounts()...) + problems = append(problems, m.unknownDirRefs()...) for i, r := range m.Resources { id, _ := r["id"].(string)