From d43fbc202c8f00c964bfdf644e5ad1e3412b34e4 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:56:43 +0000 Subject: [PATCH 1/5] Let a file resource name a template file in its module's repository A file's text may live in files/ beside the manifest instead of inline, so modules can move their files out of module.json (ADR 0349 clause 2, issue 654). The build packs files/ into the module's declared archive and reads each template back from the packed bytes, proven by the digest, into content: the manifest the mesh holds, composition and the node-engine stay as they are. module check, the merge gate and module add read templates from the tree beside the manifest, so a change to a template alone is judged. A file naming both template and content, a template outside files/, a missing one, and a held manifest that still names one are refused by name. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/check.go | 5 +++++ cmd/mesh-controller/merge_gate.go | 5 +++++ cmd/mesh-controller/modules.go | 5 +++++ internal/builder/builder.go | 14 +++++++++++++- internal/builder/reuse_test.go | 16 ++++++++++++++-- internal/catalogue/build.go | 24 ++++++++++++++++++++++++ internal/catalogue/manifest.go | 3 +++ internal/inventory/catalogue.go | 5 +++++ 8 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..64b04429 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -55,6 +55,11 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { continue } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text read from the tree beside the manifest, and judged as inline content is + // (novox/hq ADR 0349 §2): a missing template is refused by name. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { fmt.Fprintf(out, "%s: %v\n", path, err) failed++ diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..78b5332a 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -576,6 +576,11 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve return err } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text from the tree beside the manifest, as the build reads it back from the archive it + // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { if inScope(dir) { failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..2a6a0a48 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "os" + "path/filepath" "sort" "strings" "time" @@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } + // A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2). + if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil { + return err + } from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self) if err != nil { return err diff --git a/internal/builder/builder.go b/internal/builder/builder.go index e819ea41..c2c1cbf9 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -268,7 +268,14 @@ func build(ctx context.Context, run Runner, publish Publisher, } // What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is // where a changed definition is first read, so it is refused here before anything is made of it. - if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 { + // Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the + // module's directory lacks is refused here, before anything is built. + judged, err := catalogue.WithTemplatesFrom(manifest, within) + if err != nil { + say("manifest", "INVALID: %v", err) + return Result{}, err + } + if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 { err := fmt.Errorf("%s", strings.Join(problems, "; ")) say("manifest", "INVALID: %v", err) return Result{}, err @@ -410,6 +417,11 @@ func build(ctx context.Context, run Runner, publish Publisher, } } + // Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2). + if built, err = withTemplateTexts(manifest, within, built); err != nil { + say("templates", "FAILED: %v", err) + return Result{}, err + } resolved, err := manifest.Resolve(built) if err != nil { say("resolve", "FAILED: %v", err) diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go index e3e4ec24..3b3d86e4 100644 --- a/internal/builder/reuse_test.go +++ b/internal/builder/reuse_test.go @@ -73,8 +73,14 @@ const reusedModule = `{"module":"meshboard","version":"1", // aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk. func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) { t.Helper() - r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule, - "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}} + return aModuleWith(t, tree, map[string]string{"mod/" + ManifestName: reusedModule, + "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}) +} + +// aModuleWith is aModuleAt with the repository's files given. +func aModuleWith(t *testing.T, tree string, contents map[string]string) (*recorded, Runner, string) { + t.Helper() + r := &recorded{tree: tree, contents: contents} run := func(ctx context.Context, dir, name string, args ...string) (string, error) { if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" { r.ran = append(r.ran, name+" "+strings.Join(args, " ")) @@ -100,6 +106,12 @@ var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { t.Helper() r, run, workspace := aModuleAt(t, tree) + return aCheckBuiltBy(t, scratch, r, run, workspace) +} + +// aCheckBuiltBy is aCheckBuilt of the repository given. +func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { + t.Helper() spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Ref: "main"}}} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 8341916b..6fa59eb0 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -35,6 +35,10 @@ type Built struct { // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. Launchers map[string]string + // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the + // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on + // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. + Texts map[string]string `json:"-"` } // Resolve fills a manifest's resources in from what was built. @@ -71,6 +75,26 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) } + // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into + // the file's content, so what the mesh holds and composes is what the same text inline would be. + if m.Templated() { + name := TemplatesArtifact(m) + texts := by[name].Texts + if name == "" || texts == nil { + return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ + "archive it packs %s/ into (%q): the builder that built it does not read templates", + m.Module, TemplatesDir, name) + } + filled, err := fillTemplates(m.Module, m.Resources, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, TemplatesDir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + m.Resources = filled + } + out := m out.Build = nil out.Resources = nil diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 4dae3034..39dc1ba2 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -1853,6 +1853,9 @@ func ParseManifest(raw []byte) (Manifest, error) { } } problems = append(problems, m.Build.problems(m.Module)...) + // A file's text in a template file of the module's repository (novox/hq ADR 0349 §2): only a file, never beside + // content, only under files/, and packed by the module's build. + problems = append(problems, templateProblems(m)...) problems = append(problems, undeliveredBundles(m)...) // **What provides the artifact store cannot be delivered through it** (novox/hq 04-ISSUES/029). // diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6ce3740c..6cd27eb2 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -117,6 +117,11 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr } } } + // **The manifest the mesh holds carries each file's text** (novox/hq ADR 0349 §2): one still naming a template + // would compose a file with no text, so it is refused by name — a build by a builder that predates templates. + if why := catalogue.HeldTemplateProblem(m); why != "" { + return fmt.Errorf("%s is not registered: %s", m.Module, why) + } raw, err := json.Marshal(m) if err != nil { return err -- 2.54.0 From 1ce4ab8c2a915445e16d7b99b9772520e6144dc5 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:57:33 +0000 Subject: [PATCH 2/5] Let a file resource name a template file in its module's repository: the templates and their tests Co-Authored-By: Claude Opus 5.5 --- internal/builder/template.go | 94 +++++++++++++++++++ internal/builder/template_test.go | 136 ++++++++++++++++++++++++++++ internal/inventory/template_test.go | 25 +++++ 3 files changed, 255 insertions(+) create mode 100644 internal/builder/template.go create mode 100644 internal/builder/template_test.go create mode 100644 internal/inventory/template_test.go diff --git a/internal/builder/template.go b/internal/builder/template.go new file mode 100644 index 00000000..1138b566 --- /dev/null +++ b/internal/builder/template.go @@ -0,0 +1,94 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// withTemplateTexts is what was built with each template's text, for a module whose files name templates +// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed +// bytes, by its path inside the archive. Resolve reads them into the files' content. +// +// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it, +// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, +// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, +// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused. +func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) { + if !manifest.Templated() { + return built, nil + } + name := catalogue.TemplatesArtifact(manifest) + at := -1 + for i, b := range built { + if b.Name == name { + at = i + } + } + if name == "" || at < 0 { + return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from", + manifest.Module, catalogue.TemplatesDir) + } + body, err := pack(filepath.Join(within, catalogue.TemplatesDir)) + if err != nil { + return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module, + catalogue.TemplatesDir, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if built[at].Digest != digest { + return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+ + "templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir, + digest, name, orNone(built[at].Digest)) + } + texts, err := unpackTexts(body) + if err != nil { + return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err) + } + out := append([]catalogue.Built(nil), built...) + out[at].Texts = texts + return out, nil +} + +// unpackTexts is every file of a packed archive (pack), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + texts[header.Name] = string(text) + } +} + +func orNone(digest string) string { + if digest == "" { + return "no digest" + } + return digest +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go new file mode 100644 index 00000000..40ff2777 --- /dev/null +++ b/internal/builder/template_test.go @@ -0,0 +1,136 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue +// 654): the build packs files/ into the module's templates archive and reads each template's text back from the +// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline. + +const templatedModule = `{"module":"meshboard","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}, + {"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, + {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"}, + {"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}` + +var templatedFiles = map[string]string{ + "Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n", + "files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing", +} + +func contentOf(t *testing.T, m catalogue.Manifest, id string) string { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if _, has := r[catalogue.TemplateField]; has { + t.Fatalf("%s still names its template in the built manifest: %v", id, r) + } + text, ok := r["content"].(string) + if !ok { + t.Fatalf("%s carries no content: %v", id, r) + } + return text + } + } + t.Fatalf("no resource %s in %v", id, m.Resources) + return "" +} + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back +// from the tree packed again, which is the archive only when it packs to the digest taken. +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +// The tree packed again must be the archive taken; text read from anything else is not the archive's. +func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(templatedModule)) + if err != nil { + t.Fatal(err) + } + within := writeTree(t, t.TempDir(), templatedFiles) + built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}} + if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err) + } +} + +// writeTree writes files under dir and answers dir. +func writeTree(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for name, body := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} diff --git a/internal/inventory/template_test.go b/internal/inventory/template_test.go new file mode 100644 index 00000000..b9f58cc8 --- /dev/null +++ b/internal/inventory/template_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The manifest the mesh holds carries each file's text (novox/hq ADR 0349 §2): one still naming a template — a +// build by a builder that predates templates, or a manifest handed over unbuilt — would compose a file with no +// text, and is refused by name. +func TestAManifestStillNamingATemplateIsNotRegistered(t *testing.T) { + inv := ForTest(t) + m := catalogue.Manifest{Module: "notifier", Version: "1", Resources: []map[string]any{ + {"id": "conf", "type": "file", "path": "/etc/notifier.conf", "template": "files/notifier.conf"}}} + err := inv.RegisterModule(t.Context(), m, Source{}) + if err == nil || !strings.Contains(err.Error(), "files/notifier.conf") { + t.Fatalf("a manifest naming a template was registered: %v", err) + } + m.Resources[0] = map[string]any{"id": "conf", "type": "file", "path": "/etc/notifier.conf", "content": "x"} + if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { + t.Fatalf("the same manifest with its text was refused: %v", err) + } +} -- 2.54.0 From 276f6755abc1401cf0cd04e55f7565c2e4d08879 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:58:16 +0000 Subject: [PATCH 3/5] Let a file resource name a template file in its module's repository: the catalogue's reading of templates Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/template.go | 274 +++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 internal/catalogue/template.go diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go new file mode 100644 index 00000000..ac785e2a --- /dev/null +++ b/internal/catalogue/template.go @@ -0,0 +1,274 @@ +package catalogue + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "unicode/utf8" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654). +// +// **Two forms, one composition.** A file resource says its text inline (`content`) or names a template file +// beside its manifest (`"template": "files/"`), never both. The module's build packs `files/` into an +// archive artifact of its own — an `archive` artifact built `from: "files"` — and resolving the build reads each +// template's text back from that packed archive into `content` and drops `template`. So the manifest the mesh +// holds, composition and the node-engine are exactly as they were: a module with its text in `files/` composes +// byte-identically to the same module with it inline. Where there is no build — `module check`, the merge gate, +// `module add` — the text is read from the repository tree beside the manifest (WithTemplatesFrom). +// +// **Read back by the builder, carried in the resolved manifest.** The builder reads each template from the very +// bytes it hashed into the archive's digest, so the text the mesh holds is the archive's. The controller +// registers the resolved manifest as the build result carries it and fetches nothing from the artifact store; a +// manifest that still names a template when it would be held — built by a builder that predates templates — is +// refused at registration (HeldTemplateProblem). + +// TemplateField is the key a file resource names its template file by. +const TemplateField = "template" + +// TemplatesDir is the module's directory its template files are kept in, and the directory its templates archive +// is built from. +const TemplatesDir = "files" + +// Templated is whether any of the manifest's resources names a template. +func (m Manifest) Templated() bool { + for _, r := range m.Resources { + if _, has := r[TemplateField]; has { + return true + } + } + return false +} + +// TemplatesArtifact is the name of the archive artifact the module packs its templates into — the one built from +// files/ — or "" when it declares none, or more than one (both refused by templateProblems for a module naming a +// template). +func TemplatesArtifact(m Manifest) string { + found := templatesArtifacts(m) + if len(found) != 1 { + return "" + } + return found[0] +} + +func templatesArtifacts(m Manifest) []string { + if m.Build == nil { + return nil + } + var found []string + for _, a := range m.Build.Artifacts { + if a.Kind == ArtifactArchive && a.URL == "" && cleanFrom(a.From) == TemplatesDir { + found = append(found, a.Name) + } + } + return found +} + +// cleanFrom is an artifact's directory as one relative path: "./files/" is "files". +func cleanFrom(from string) string { + c := path.Clean(strings.TrimSpace(from)) + return strings.TrimPrefix(c, "./") +} + +// templateOf is the template a resource names, whether it names one, and what is wrong with it ("" when nothing). +func templateOf(module string, r map[string]any) (string, bool, string) { + raw, has := r[TemplateField] + if !has { + return "", false, "" + } + id := fmt.Sprint(r["id"]) + if t := fmt.Sprint(r["type"]); t != "file" { + return "", true, fmt.Sprintf("%s: %s is a %s and names a template; only a file takes its text from one "+ + "(novox/hq ADR 0349 §2)", module, id, t) + } + name, ok := raw.(string) + if !ok || strings.TrimSpace(name) == "" { + return "", true, fmt.Sprintf("%s: %s names a template that is not a path: %v; a template is %q", + module, id, raw, TemplatesDir+"/") + } + if _, both := r["content"]; both { + return name, true, fmt.Sprintf("%s: %s names the template %q and says its content too; a file's text is "+ + "in one place, the template or the content, never both (novox/hq ADR 0349 §2)", module, id, name) + } + if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, TemplatesDir+"/") || + strings.Contains(name, "\\") { + return name, true, fmt.Sprintf("%s: %s names the template %q, which is not a file under %s/ of its own "+ + "repository: a template is %q, a plain relative path, and the build packs only %s/", + module, id, name, TemplatesDir, TemplatesDir+"/", TemplatesDir) + } + return name, true, "" +} + +// templateProblems is what is wrong with the manifest's templates, as written: a template on something not a file, +// beside content, outside files/, or a module naming templates whose build packs no archive from files/. Whether +// each template exists is judged where a tree is (ReadTemplates) or the archive's text is (Resolve). +func templateProblems(m Manifest) []string { + var problems []string + templated := false + for _, r := range m.Resources { + _, has, wrong := templateOf(m.Module, r) + templated = templated || has + if wrong != "" { + problems = append(problems, wrong) + } + } + if !templated { + return problems + } + switch found := templatesArtifacts(m); len(found) { + case 0: + problems = append(problems, fmt.Sprintf("%s names templates and its build packs no archive from %s/: "+ + "declare an artifact {\"name\": \"files\", \"kind\": \"archive\", \"from\": %q} (novox/hq ADR 0349 §2)", + m.Module, TemplatesDir, TemplatesDir)) + case 1: + default: + sort.Strings(found) + problems = append(problems, fmt.Sprintf("%s packs %s/ into %s; its templates are read from one archive", + m.Module, TemplatesDir, strings.Join(found, " and "))) + } + return problems +} + +// fillTemplates is resources with each template's text as its content and the template key dropped: text says the +// text of a template by its path ("files/"), and whether it has one. A template it has no text for, or text +// that is not UTF-8, is refused by name. Every resource is copied; the manifest's own are left as they were. +func fillTemplates(module string, resources []map[string]any, text func(string) (string, bool), + where string) ([]map[string]any, error) { + out := make([]map[string]any, 0, len(resources)) + var problems []string + for _, r := range resources { + name, has, wrong := templateOf(module, r) + if !has { + out = append(out, r) + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + body, ok := text(name) + if !ok { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and %s holds no such file", + module, r["id"], name, where)) + continue + } + if !utf8.ValidString(body) { + // A manifest is JSON: text that is not UTF-8 would be changed on the way, silently. + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not UTF-8 text; a "+ + "file's text is carried as JSON and would not reach the machine as written", module, r["id"], name)) + continue + } + filled := make(map[string]any, len(r)) + for k, v := range r { + filled[k] = v + } + delete(filled, TemplateField) + filled["content"] = body + out = append(out, filled) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return out, nil +} + +// ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository +// tree, by template path. A missing template, or one that is not a plain file (a symlink is refused by the build's +// packing too), is refused by name. +func ReadTemplates(m Manifest, dir string) (map[string]string, error) { + texts := map[string]string{} + var problems []string + for _, r := range m.Resources { + name, has, wrong := templateOf(m.Module, r) + if !has { + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + if _, read := texts[name]; read { + continue + } + full := filepath.Join(dir, filepath.FromSlash(name)) + info, err := os.Lstat(full) + switch { + case errors.Is(err, fs.ErrNotExist): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+ + "has no such file", m.Module, r["id"], name)) + continue + case err != nil: + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + case !info.Mode().IsRegular(): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not a plain file; "+ + "the build packs only files", m.Module, r["id"], name)) + continue + } + body, err := os.ReadFile(full) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + } + texts[name] = string(body) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return texts, nil +} + +// WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository +// tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read +// again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges +// inline content. A manifest naming no template is returned as it is. +func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { + if !m.Templated() { + return m, nil + } + texts, err := ReadTemplates(m, dir) + if err != nil { + return Manifest{}, err + } + filled, err := fillTemplates(m.Module, m.Resources, func(name string) (string, bool) { + t, ok := texts[name] + return t, ok + }, "the module's directory") + if err != nil { + return Manifest{}, err + } + out := m + out.Resources = filled + raw, err := json.Marshal(out) + if err != nil { + return Manifest{}, err + } + return ParseManifest(raw) +} + +// HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the +// manifest the mesh holds carries every file's text, which a build reads back from its templates archive. One +// naming a template was resolved by a builder that does not read templates, and would compose files with no text. +func HeldTemplateProblem(m Manifest) string { + var named []string + for _, r := range m.Resources { + if t, has := r[TemplateField]; has { + named = append(named, fmt.Sprintf("%v (%v)", r["id"], t)) + } + } + if len(named) == 0 { + return "" + } + return fmt.Sprintf("%s still names the template(s) of %s: the manifest the mesh holds carries each file's "+ + "text, read back from the build's templates archive — this one was resolved by a builder that does not "+ + "read templates, or not built at all (novox/hq ADR 0349 §2)", m.Module, strings.Join(named, ", ")) +} -- 2.54.0 From f45e0a1de50828323532084196696a25a16da4f9 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:59:22 +0000 Subject: [PATCH 4/5] Let a file resource name a template file in its module's repository: composition, check and gate tests Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/template_test.go | 177 ++++++++++++++++++ internal/catalogue/template_test.go | 258 +++++++++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 cmd/mesh-controller/template_test.go create mode 100644 internal/catalogue/template_test.go diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go new file mode 100644 index 00000000..60ca0e74 --- /dev/null +++ b/cmd/mesh-controller/template_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a +// template alone is judged as a change to its module. + +const templatedNotifier = `{"module":"notifier","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}` + +// inlineNotifier is the same module as the mesh holds it: built, its text inline. +const inlineNotifier = `{"module":"notifier","version":"1", + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}` + +// aTemplatedModule writes a module's directory: its manifest and its files/. +func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "module.json") + if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + for name, text := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { + good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + var out bytes.Buffer + if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") { + t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String()) + } + + // A change to the template alone, to text the controller refuses inline, is refused the same. + bad := aTemplatedModule(t, t.TempDir(), templatedNotifier, + map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"}) + out.Reset() + if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") { + t.Fatalf("a template whose text is refused inline passed:\n%s", out.String()) + } + + // And a template the tree lacks, by name. + missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil) + out.Reset() + if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") { + t.Fatalf("a missing template was not refused by name:\n%s", out.String()) + } +} + +// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline. +func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + "notifier": inlineNotifier, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", name); err != nil { + t.Fatalf("assign %s: %v", name, err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +// aTemplatedTree is the catalogue's checkout with the notifier's text in files/. +func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string { + t.Helper() + tree := aTree(t, withEdit(manifests, "notifier", "")) + files := map[string]string{} + if has { + files["files/notifier.conf"] = text + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, files) + return tree +} + +const notifierTemplate = "modules/notifier/files/notifier.conf" + +func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { + f, manifests := templatedMesh(t) + + // The module moved to its template, the same text: nothing changes on any machine. + v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", + notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "" { + t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) + } + + // The template alone changed: its module changed, and composes. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) + } + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { + t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) + } + + // The template alone changed to text the controller refuses: the pull request fails, naming the module. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { + t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) + } + + // And the template removed from the tree fails, naming it. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { + t.Fatalf("a missing template passed the gate:\n%s", v.Report()) + } +} + +// What the gate composes for the module in its template form is what the mesh composes for it inline. +func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + m, err = catalogue.WithTemplatesFrom(m, dir) + if err != nil { + t.Fatal(err) + } + resolved, err := standInBuild(m) + if err != nil { + t.Fatal(err) + } + inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) + if err != nil { + t.Fatal(err) + } + if builtAlike(resolved) != builtAlike(inline) { + got, _ := json.Marshal(resolved) + t.Fatalf("the templated module resolves to %s, not its inline twin", got) + } +} diff --git a/internal/catalogue/template_test.go b/internal/catalogue/template_test.go new file mode 100644 index 00000000..85c81896 --- /dev/null +++ b/internal/catalogue/template_test.go @@ -0,0 +1,258 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// A file resource may name a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package +// T1, issue 654): the build packs files/ into an archive, and resolving the build reads each template's text back +// into content, so the mesh holds, composes and sends exactly what it would for the same text inline. + +// The two forms of one module: its text inline, and its text in files/. +const ( + inlineTwin = `{"module":"tidy","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"conf","type":"directory","path":"${dir:conf}"}, + {"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","content":"` + mainText + `"}, + {"id":"extra","type":"file","path":"${dir:conf}/extra.conf","content":"` + extraText + `"}]}` + templatedTwin = `{"module":"tidy","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"conf","type":"directory","path":"${dir:conf}"}, + {"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","template":"files/tidy.conf"}, + {"id":"extra","type":"file","path":"${dir:conf}/extra.conf","template":"files/deeper/extra.conf"}]}` + mainText = `# tidy\nroot = ${dir:conf}\nquoted = \"yes\" é\n` + extraText = `one\ntwo\n` +) + +// templateTexts is what a build reads back from the packed files/ archive: by path inside it. +func templateTexts(t *testing.T) map[string]string { + t.Helper() + var m struct{ Main, Extra string } + if err := json.Unmarshal([]byte(`{"Main":"`+mainText+`","Extra":"`+extraText+`"}`), &m); err != nil { + t.Fatal(err) + } + return map[string]string{"tidy.conf": m.Main, "deeper/extra.conf": m.Extra, "unused.conf": "not named"} +} + +func builtFiles(texts map[string]string) []Built { + return []Built{{Name: "files", Kind: ArtifactArchive, + Reference: ArtifactStoreScheme + "tidy/files/blobs/sha256:" + strings.Repeat("c", 64), + Digest: "sha256:" + strings.Repeat("c", 64), Texts: texts}} +} + +// held is a manifest as registration holds it: resolved with what was built, then read again strictly. +func held(t *testing.T, raw string, built []Built) Manifest { + t.Helper() + m, err := ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + resolved, err := m.Resolve(built) + if err != nil { + t.Fatal(err) + } + again, err := json.Marshal(resolved) + if err != nil { + t.Fatal(err) + } + out, err := ParseManifest(again) + if err != nil { + t.Fatal(err) + } + if problem := HeldTemplateProblem(out); problem != "" { + t.Fatal(problem) + } + return out +} + +func composed(t *testing.T, m Manifest) string { + t.Helper() + got, err := Resolve(shelf(m), []string{m.Module}, workstation(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := json.Marshal(mustDeclare(t, got)) + if err != nil { + t.Fatal(err) + } + return string(out) +} + +func TestATemplatedModuleComposesByteIdenticallyToItsInlineTwin(t *testing.T) { + inline := composed(t, held(t, inlineTwin, builtFiles(nil))) + templated := composed(t, held(t, templatedTwin, builtFiles(templateTexts(t)))) + if inline != templated { + t.Fatalf("the two forms compose differently:\ninline: %s\ntemplated: %s", inline, templated) + } + if !strings.Contains(templated, "quoted = \\\"yes\\\"") || strings.Contains(templated, `"template"`) { + t.Fatalf("the composition does not carry the template's text as content: %s", templated) + } +} + +// What `module check`, the merge gate and `module add` judge: the text read from the tree beside the manifest. +func TestATemplatedModuleReadFromItsTreeComposesByteIdenticallyToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + for name, text := range templateTexts(t) { + full := filepath.Join(dir, "files", filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + filled, err := WithTemplatesFrom(m, dir) + if err != nil { + t.Fatal(err) + } + if filled.Templated() { + t.Fatal("a manifest read from its tree still names a template") + } + resolved, err := filled.Resolve(builtFiles(nil)) + if err != nil { + t.Fatal(err) + } + if got, want := composed(t, resolved), composed(t, held(t, inlineTwin, builtFiles(nil))); got != want { + t.Fatalf("read from the tree, the module composes differently:\ninline: %s\ntemplated: %s", want, got) + } +} + +func TestAFileNamingBothATemplateAndContentIsRefused(t *testing.T) { + raw := strings.Replace(templatedTwin, `"template":"files/tidy.conf"`, + `"template":"files/tidy.conf","content":"x"`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "never both") { + t.Fatalf("a file with both a template and content was not refused by name: %v", err) + } +} + +func TestATemplateOutsideFilesIsRefusedByName(t *testing.T) { + for _, bad := range []string{"../tidy.conf", "/etc/tidy.conf", "files", "files/", "files/../tidy.conf", + "files/./tidy.conf", "other/tidy.conf", "tidy.conf", "files//tidy.conf", `files\\tidy.conf`} { + raw := strings.Replace(templatedTwin, `"files/tidy.conf"`, `"`+bad+`"`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "not a file under files/") { + t.Errorf("the template %q was not refused by name: %v", bad, err) + } + } +} + +func TestATemplateOnAResourceThatIsNotAFileIsRefused(t *testing.T) { + raw := strings.Replace(templatedTwin, `{"id":"conf","type":"directory","path":"${dir:conf}"}`, + `{"id":"conf","type":"directory","path":"${dir:conf}","template":"files/tidy.conf"}`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "conf is a directory and names a template") { + t.Fatalf("a directory naming a template was not refused: %v", err) + } +} + +func TestATemplatedModuleWhoseBuildPacksNoFilesIsRefused(t *testing.T) { + for _, build := range []string{ + ``, + `"build":{"artifacts":[{"name":"look","kind":"archive","from":"theme"}]},`, + `"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"},{"name":"b","kind":"archive","from":"./files/"}]},`, + } { + raw := strings.Replace(templatedTwin, + `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, build, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "files/") { + t.Errorf("a templated module with the build %q was not refused: %v", build, err) + } + } +} + +func TestAMissingTemplateIsRefusedByName(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + // In the tree beside the manifest. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + _, err = WithTemplatesFrom(m, dir) + if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") { + t.Fatalf("a template missing from the tree was not refused by name: %v", err) + } + // And in what the build read back from its archive. + texts := templateTexts(t) + delete(texts, "deeper/extra.conf") + _, err = m.Resolve(builtFiles(texts)) + if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") { + t.Fatalf("a template missing from the archive was not refused by name: %v", err) + } +} + +func TestATemplateThatIsASymlinkIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "files", "deeper"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "elsewhere"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, "files", "deeper", "extra.conf")); err != nil { + t.Fatal(err) + } + if _, err := WithTemplatesFrom(m, dir); err == nil || !strings.Contains(err.Error(), "not a plain file") { + t.Fatalf("a template that is a symlink was not refused: %v", err) + } +} + +func TestAResolveWithoutTheArchivesTextIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + if _, err := m.Resolve(builtFiles(nil)); err == nil || !strings.Contains(err.Error(), "files") { + t.Fatalf("a templated module resolved with no text read back from its archive: %v", err) + } +} + +func TestATemplateThatIsNotUTF8IsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + texts := templateTexts(t) + texts["tidy.conf"] = "\xff\xfe" + if _, err := m.Resolve(builtFiles(texts)); err == nil || !strings.Contains(err.Error(), "UTF-8") { + t.Fatalf("a template that is not UTF-8 was resolved: %v", err) + } +} + +// What registration refuses: a manifest to be held that still names a template — one resolved by a builder that +// predates templates would otherwise compose files with no text. +func TestAManifestStillNamingATemplateIsNotHeld(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + if problem := HeldTemplateProblem(m); !strings.Contains(problem, "main") || !strings.Contains(problem, "files/tidy.conf") { + t.Fatalf("a manifest naming a template would be held: %q", problem) + } + if problem := HeldTemplateProblem(held(t, inlineTwin, builtFiles(nil))); problem != "" { + t.Fatalf("an inline manifest is refused: %s", problem) + } +} -- 2.54.0 From def3ccefe3c8f26a4bb2dc6f95ba6efd9949bb6d Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:51:48 +0000 Subject: [PATCH 5/5] Refuse templates read through a symlink anywhere under files/ A symlinked files/ or a symlinked directory under it let module check, the merge gate and module add read a template from outside the repository tree, and a symlink no template names passed them while the build's packing refuses it (review of mesh-controller#313). Every entry under files/ must now be a plain directory or file, and templates are read through an os.Root of the module's directory. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/template_test.go | 27 +++++++++++ internal/catalogue/template.go | 72 +++++++++++++++++++++++++--- internal/catalogue/template_test.go | 65 +++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go index 60ca0e74..edcc66a9 100644 --- a/cmd/mesh-controller/template_test.go +++ b/cmd/mesh-controller/template_test.go @@ -62,6 +62,18 @@ func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { t.Fatalf("a template whose text is refused inline passed:\n%s", out.String()) } + // A symlink under files/ that no template names: the build's packing refuses it, so the check does too. + linked := t.TempDir() + aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil { + t.Fatal(err) + } + out.Reset() + if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil || + !strings.Contains(out.String(), "files/elsewhere is a symlink") { + t.Fatalf("a symlink under files/ passed the check:\n%s", out.String()) + } + // And a template the tree lacks, by name. missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil) out.Reset() @@ -139,6 +151,21 @@ func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) } + // A symlinked files/ fails: nothing is read from outside the tree. + tree := aTree(t, withEdit(manifests, "notifier", "")) + elsewhere := t.TempDir() + if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { + t.Fatal(err) + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) + if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { + t.Fatal(err) + } + v = gateJudged(t, f, tree, notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { + t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) + } + // And the template removed from the tree fails, naming it. v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go index ac785e2a..2a363bf2 100644 --- a/internal/catalogue/template.go +++ b/internal/catalogue/template.go @@ -180,11 +180,18 @@ func fillTemplates(module string, resources []map[string]any, text func(string) } // ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository -// tree, by template path. A missing template, or one that is not a plain file (a symlink is refused by the build's -// packing too), is refused by name. +// tree, by template path. Refused by name: a template the directory lacks, and anything under files/ the build's +// packing refuses — files/ itself, or any directory or file under it, that is a symlink or not a plain file — so +// nothing judged here can be read from outside the tree, and no tree passes here that the build would refuse. Each +// template is read through an os.Root of the module's directory besides, which no path element can leave. func ReadTemplates(m Manifest, dir string) (map[string]string, error) { + problems := templatesTreeProblems(m.Module, dir) + root, err := os.OpenRoot(dir) + if err != nil { + return nil, fmt.Errorf("%s: the module's directory cannot be opened to read its templates: %w", m.Module, err) + } + defer root.Close() texts := map[string]string{} - var problems []string for _, r := range m.Resources { name, has, wrong := templateOf(m.Module, r) if !has { @@ -197,8 +204,7 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { if _, read := texts[name]; read { continue } - full := filepath.Join(dir, filepath.FromSlash(name)) - info, err := os.Lstat(full) + info, err := root.Lstat(name) switch { case errors.Is(err, fs.ErrNotExist): problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+ @@ -213,7 +219,7 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { "the build packs only files", m.Module, r["id"], name)) continue } - body, err := os.ReadFile(full) + body, err := root.ReadFile(name) if err != nil { problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", m.Module, r["id"], name, err)) @@ -227,6 +233,60 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { return texts, nil } +// templatesTreeProblems is everything under the module's files/ that the build's packing refuses: files/ itself not +// a plain directory, and any entry under it that is neither a directory nor a plain file — a symlink, to a file or a +// directory, a device, a pipe. Each said by its path. A files/ that does not exist is no problem here: each template +// it would have held is refused as missing. +func templatesTreeProblems(module, dir string) []string { + top := filepath.Join(dir, TemplatesDir) + info, err := os.Lstat(top) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return []string{fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)} + } + if !info.IsDir() { + return []string{fmt.Sprintf("%s: %s/ is %s, not a directory of the module's own; its templates are read "+ + "only from inside the repository, and the build packs only directories and files", module, TemplatesDir, + kindOf(info.Mode()))} + } + var problems []string + err = filepath.WalkDir(top, func(at string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Type().IsRegular() { + return nil + } + rel, _ := filepath.Rel(dir, at) + problems = append(problems, fmt.Sprintf("%s: %s is %s; the build packs only directories and files, so "+ + "%s/ holds nothing else", module, filepath.ToSlash(rel), kindOf(d.Type()), TemplatesDir)) + return nil + }) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)) + } + return problems +} + +// kindOf says what a file that is not a plain file is. +func kindOf(mode fs.FileMode) string { + switch { + case mode&fs.ModeSymlink != 0: + return "a symlink" + case mode&fs.ModeDevice != 0: + return "a device" + case mode&fs.ModeNamedPipe != 0: + return "a named pipe" + case mode&fs.ModeSocket != 0: + return "a socket" + case mode.IsDir(): + return "a directory" + } + return "not a plain file" +} + // WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository // tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read // again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges diff --git a/internal/catalogue/template_test.go b/internal/catalogue/template_test.go index 85c81896..a439acb0 100644 --- a/internal/catalogue/template_test.go +++ b/internal/catalogue/template_test.go @@ -256,3 +256,68 @@ func TestAManifestStillNamingATemplateIsNotHeld(t *testing.T) { t.Fatalf("an inline manifest is refused: %s", problem) } } + +// A symlinked files/, or a symlinked directory under it, would read a file outside the tree: refused by name, +// as is any symlink or special file under files/ that no template names, which the build's packing refuses. +func TestASymlinkAnywhereUnderFilesIsRefusedByName(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(outside, "deeper"), 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"tidy.conf", "deeper/extra.conf", "secret"} { + if err := os.WriteFile(filepath.Join(outside, filepath.FromSlash(name)), []byte("outside"), 0o644); err != nil { + t.Fatal(err) + } + } + write := func(t *testing.T, dir, name string) { + t.Helper() + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte("inside"), 0o644); err != nil { + t.Fatal(err) + } + } + link := func(t *testing.T, target, at string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(at), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, at); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + make func(t *testing.T, dir string) + names string + }{ + {"files/ itself", func(t *testing.T, dir string) { + link(t, outside, filepath.Join(dir, "files")) + }, "files/ is a symlink"}, + {"a directory under files/", func(t *testing.T, dir string) { + write(t, dir, "files/tidy.conf") + link(t, filepath.Join(outside, "deeper"), filepath.Join(dir, "files", "deeper")) + }, "files/deeper is a symlink"}, + {"a file no template names", func(t *testing.T, dir string) { + write(t, dir, "files/tidy.conf") + write(t, dir, "files/deeper/extra.conf") + link(t, filepath.Join(outside, "secret"), filepath.Join(dir, "files", "unnamed")) + }, "files/unnamed is a symlink"}, + } { + dir := t.TempDir() + c.make(t, dir) + _, err := WithTemplatesFrom(m, dir) + if err == nil || !strings.Contains(err.Error(), c.names) { + t.Errorf("a symlinked %s was not refused by name: %v", c.name, err) + } + if err != nil && strings.Contains(err.Error(), "outside") { + t.Errorf("a symlinked %s was read: %v", c.name, err) + } + } +} -- 2.54.0