From 4d1b81b6cb5367e97be8d4c5332a089b7177ad27 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 10 Oct 2026 02:47:35 +0200 Subject: [PATCH] Say what a build was made from, and hand an image compiling Go only that (hq ADR 0267, issue 363) A merge to the controller's repository moved the route proxy and the build seat's holder whatever it changed, because nothing said which files their builds read. A build of a trunk commit now says its build source per repository: a Go program's import closure, an archive's directory, an image's recipe and the package it names in the new 'compiles' field. That image is built from its build source alone, so a recipe reading past it fails by name, and its fingerprint is over what it was handed. --- cmd/mesh-builder/main.go | 3 + internal/builder/build_source_test.go | 176 ++++++++++ internal/builder/builder.go | 92 ++++- internal/builder/gosource.go | 485 ++++++++++++++++++++++++++ internal/builder/gosource_test.go | 288 +++++++++++++++ internal/builder/source.go | 126 ++++++- internal/catalogue/build.go | 11 + internal/catalogue/build_test.go | 21 ++ internal/catalogue/manifest.go | 10 + internal/link/build.go | 16 + 10 files changed, 1222 insertions(+), 6 deletions(-) create mode 100644 internal/builder/build_source_test.go create mode 100644 internal/builder/gosource.go create mode 100644 internal/builder/gosource_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 672e6d8e..c49fc8cb 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -306,6 +306,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri for _, r := range built.Read { result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref}) } + for _, s := range built.Sources { + result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) + } say("built", built.Manifest.Module+" from "+short(built.Commit)) } } diff --git a/internal/builder/build_source_test.go b/internal/builder/build_source_test.go new file mode 100644 index 00000000..56a2cd1d --- /dev/null +++ b/internal/builder/build_source_test.go @@ -0,0 +1,176 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// What a build says it was made from, as files (novox/hq ADR 0267), and what a build compiling a Go +// program is handed. + +// onTrunk answers the trunk's questions as a clone of a commit on main would, or off it. +type onTrunk struct { + *recorded + off bool +} + +func (o onTrunk) run(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" { + return "origin/main\n", nil + } + if name == "git" && len(args) > 0 && args[0] == "merge-base" && o.off { + return "", os.ErrNotExist + } + return compiling{o.recorded}.run(ctx, dir, name, args...) +} + +// aSharedRepository is a repository holding two programs that share a package, as the controller's does. +func aSharedRepository(readme, shared string) map[string]string { + return map[string]string{ + "go.mod": "module example.com/ctl\n\ngo 1.22\n", + "go.sum": "", + "README.md": readme, + "cmd/ctl/main.go": "package main\n\nimport _ \"example.com/ctl/internal/shared\"\n\nfunc main() {}\n", + "proxy/main.go": "package main\n\nimport _ \"example.com/ctl/internal/shared\"\n\nfunc main() {}\n", + "internal/shared/s.go": "package shared\n\nconst S = " + shared + "\n", + "internal/only/o.go": "package only\n", + } +} + +const aProxy = `{"module":"route-proxy","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile","compiles":"proxy", + "context":{"repository":"https://forge.invalid/ctl.git","ref":"main"}}, + {"name":"trust","kind":"upstream","from":"alpine@sha256:` + "3333333333333333333333333333333333333333333333333333333333333333" + `"}]}}` + +func buildTheProxy(t *testing.T, context_ map[string]string, off bool) (Result, *recorded, string) { + t.Helper() + r := &recorded{ + contents: map[string]string{"modules/route-proxy/" + ManifestName: aProxy, "modules/route-proxy/Dockerfile": "FROM scratch\nCOPY . .\n", "modules/route-proxy/README.md": "x"}, + secondary: map[string]map[string]string{"https://forge.invalid/ctl.git": context_}, + } + workspace := t.TempDir() + got, err := Build(context.Background(), onTrunk{r, off}.run, r, + "https://forge.invalid/catalogue.git", "modules/route-proxy", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + return got, r, workspace +} + +func TestAnImageCompilingGoIsHandedItsBuildSourceAndSaysIt(t *testing.T) { + got, r, workspace := buildTheProxy(t, aSharedRepository("one", "1"), false) + + // Built in the narrowed tree, which holds the program's closure and nothing else. + at := "" + for i, line := range r.ran { + if strings.HasPrefix(line, "docker build ") { + at = r.dirs[i] + } + } + if filepath.Base(at) != "narrow-server" { + t.Fatalf("docker build ran in %q, not the narrowed build source", at) + } + for file, want := range map[string]bool{"proxy/main.go": true, "internal/shared/s.go": true, "go.mod": true, + "cmd/ctl/main.go": false, "internal/only/o.go": false, "README.md": false} { + _, err := os.Stat(filepath.Join(workspace, "narrow-server", filepath.FromSlash(file))) + if (err == nil) != want { + t.Errorf("%s handed to the recipe: %v, wanted %v", file, err == nil, want) + } + } + + // Said per repository: its own, the manifest and the recipe; the context's, the closure. + if len(got.Sources) != 2 { + t.Fatalf("sources %+v", got.Sources) + } + own, ctx := got.Sources[0], got.Sources[1] + if own.Repository != "" || !slices.Equal(own.Paths, []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}) { + t.Errorf("its own build source: %+v", own) + } + if ctx.Repository != "https://forge.invalid/ctl.git" || ctx.Ref != "main" { + t.Errorf("the context's build source names %q at %q", ctx.Repository, ctx.Ref) + } + for file, want := range map[string]bool{"proxy/main.go": true, "internal/shared/s.go": true, "go.mod": true, + "cmd/ctl/main.go": false, "README.md": false} { + if SourceHolds(ctx.Paths, file) != want { + t.Errorf("the context's build source holds %s: %v, wanted %v (%v)", file, !want, want, ctx.Paths) + } + } + + // **The fingerprint is over the build source** (rule 5): a change outside it is one build, inside it another. + readme, _, _ := buildTheProxy(t, aSharedRepository("two", "1"), false) + if readme.Source != got.Source { + t.Errorf("a README of the context changed the fingerprint: %s %s", got.Source, readme.Source) + } + shared, _, _ := buildTheProxy(t, aSharedRepository("one", "2"), false) + if shared.Source == got.Source { + t.Error("a change to the program's closure kept its fingerprint") + } +} + +// A build off the trunk says no build source: the planner maps a merge onto the trunk's. +func TestABuildOffTheTrunkSaysNoBuildSource(t *testing.T) { + got, _, _ := buildTheProxy(t, aSharedRepository("one", "1"), true) + if len(got.Sources) != 0 { + t.Fatalf("a build off the trunk said %+v", got.Sources) + } +} + +// A recipe reading past its build source fails, naming what it could not find — in the real docker build; +// here, the file is simply not in the tree it is handed, which is what makes that so. +func TestAnImageCompilingANonexistentPackageFails(t *testing.T) { + r := &recorded{ + contents: map[string]string{ManifestName: strings.Replace(aProxy, `"compiles":"proxy"`, `"compiles":"nowhere"`, 1), + "Dockerfile": "FROM scratch\n"}, + secondary: map[string]map[string]string{"https://forge.invalid/ctl.git": aSharedRepository("one", "1")}, + } + _, err := Build(context.Background(), onTrunk{r, false}.run, r, + "https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "nowhere") { + t.Fatalf("a package that is not there built: %v", err) + } +} + +// A Go bundle of a module built from its repository's root says its import closure, and the manifest; +// an image that compiles nothing it was told of leaves the module's source whole, and says none. +func TestAGoBundleSaysItsClosureAndAnUntoldImageNothing(t *testing.T) { + files := aSharedRepository("one", "1") + manifest := `{"module":"ctl","version":"1","build":{"artifacts":[ + {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]}, + "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}` + r := &recorded{contents: map[string]string{ManifestName: manifest}} + for k, v := range files { + r.contents[k] = v + } + held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)} + got, err := Build(context.Background(), onTrunk{r, false}.run, r, + "https://forge.invalid/ctl.git", "", "", t.TempDir(), held, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Sources) != 1 || got.Sources[0].Repository != "" { + t.Fatalf("sources %+v", got.Sources) + } + for file, want := range map[string]bool{"cmd/ctl/main.go": true, "internal/shared/s.go": true, ManifestName: true, + "go.sum": true, "proxy/main.go": false, "README.md": false, "internal/only/o.go": false} { + if SourceHolds(got.Sources[0].Paths, file) != want { + t.Errorf("%s: held %v, wanted %v (%v)", file, !want, want, got.Sources[0].Paths) + } + } + + untold := `{"module":"ctl","version":"1","build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}]}}` + r = &recorded{contents: map[string]string{ManifestName: untold, "Dockerfile": "FROM scratch\n"}} + got, err = Build(context.Background(), onTrunk{r, false}.run, r, + "https://forge.invalid/ctl.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Sources) != 0 { + t.Fatalf("an image compiling nothing it was told of said a build source: %+v", got.Sources) + } +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 284005a7..098ab18b 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -11,6 +11,7 @@ import ( "io" "os" "os/exec" + "path" "path/filepath" "regexp" "sort" @@ -87,6 +88,23 @@ type Result struct { // pin the build. Two builds with one fingerprint are one build, whatever digests they made // (novox/hq issue 280). Source string + + // Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the + // entries a changed file is tested against (SourceHolds). The module's own repository has an empty + // Repository. Said only for a build of a commit on the trunk, and only for a repository whose every + // artifact's build source is known — a Go program's import closure, an archive's directory, an image's + // recipe and the package it compiles; for any other, nothing is said and the whole of what the build + // sees stays its source, as before. + Sources []BuildSource +} + +// BuildSource is the build source a build read in one repository (novox/hq ADR 0267). +type BuildSource struct { + // Repository and Ref are a context's, as the manifest names it; empty for the module's own. + Repository string + Ref string + // Paths are the entries, relative to the repository's root (SourceHolds). + Paths []string } // GitCredential is the forge credential a clone may present when the server asks for one. @@ -206,6 +224,10 @@ func build(ctx context.Context, run Runner, publish Publisher, // What it is made from, for its source fingerprint: the module's own tree first. src := newSourceInputs(manifest.Module) + src.prefix = strings.Trim(filepath.ToSlash(filepath.Clean(path)), "/") + if src.prefix == "." { + src.prefix = "" + } if src.tree, err = gitTree(ctx, run, tree, path); err != nil { src.notPinned("its tree could not be named: " + err.Error()) } @@ -298,9 +320,22 @@ func build(ctx context.Context, run Runner, publish Publisher, if fingerprint == "" { say("source", "no source fingerprint: %s", orNoTree(src.unpinned)) } + // **Only a trunk build says its build source** (novox/hq ADR 0267): the planner maps the next merge onto + // the build source of the trunk's last build, and a branch's closure is not the trunk's. + var sources []BuildSource + if trunk != "" && onTrunk { + sources = src.buildSources() + for _, s := range sources { + where := "its own repository" + if s.Repository != "" { + where = s.Repository + } + say("source", "%d path(s) of %s", len(s.Paths), where) + } + } return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, - Trunk: trunk, OnTrunk: onTrunk, Branches: branches}, nil + Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil } // branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`. @@ -648,15 +683,48 @@ func one(ctx context.Context, run Runner, publish Publisher, } else if src != nil { src.contexts[a.Name] = t } - // docker build accepts -f outside the context it is given; the recipe stays exactly - // where it was read from and validated against, absolute so the working directory - // switching to the cloned context does not change which file that is. + buildDir = cloned + } + // docker build accepts -f outside the context it is given; the recipe stays exactly where it was + // read from and validated against, absolute so a context elsewhere does not change which file + // that is. + if buildDir != tree || a.Compiles != "" { absRecipe, err := filepath.Abs(filepath.Join(tree, a.From)) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err) } recipePath = absRecipe - buildDir = cloned + } + // **An image that compiles a Go program is handed its build source and nothing else** (novox/hq + // ADR 0267 rules 1 and 3): the program's import closure, read from the context it is built in, so a + // merge elsewhere in that repository is no change to it — and a recipe that copies a file outside + // it fails here, naming the file, rather than building from something no merge is mapped onto. + if a.Compiles != "" { + paths, err := GoBuildSource(buildDir, a.Compiles) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s compiles %s, whose build source cannot be read: %w", + module, a.Name, a.Compiles, err) + } + narrowed := filepath.Join(workspace, "narrow-"+a.Name) + sum, err := narrowTree(buildDir, narrowed, paths) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: handing %s its build source: %w", module, a.Name, err) + } + say("image", "%s is handed its build source: %d path(s) of %s", a.Name, len(paths), a.Compiles) + if a.Context != nil { + src.contexts[a.Name] = sum + src.readIn(*a.Context, paths) + } else { + src.ownHas(paths...) + } + buildDir = narrowed + } else if a.Context != nil { + src.readWhole(*a.Context) + } + if a.Compiles != "" || a.Context != nil { + src.ownHas(a.From) + } else { + src.ownWhole() } invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...) if a.Target != "" { @@ -708,6 +776,18 @@ func one(ctx context.Context, run Runner, publish Publisher, if src != nil { src.toolchains[a.Name] = toolchainOf(chain, base) } + // A Go program's build source is its import closure (novox/hq ADR 0267 rule 1). Not read, it is the + // module's whole directory, as before — said, so the wider plan has a reason a person can find. + if chain.Language == "go" { + if paths, err := GoBuildSource(tree, a.From); err != nil { + say("bundle", "%s's build source is its whole directory: %v", a.Name, err) + src.ownWhole() + } else { + src.ownHas(paths...) + } + } else { + src.ownWhole() + } if chain.Language == "typescript" { if own, _ := ownDependencies(tree); len(own) > 0 { src.notPinned(a.Name + " resolves packages of its own at build time") @@ -754,6 +834,7 @@ func one(ctx context.Context, run Runner, publish Publisher, // there is no Publisher call — the container itself publishes, with the credential the // build was handed. say("package", "building and publishing %s (%s)", a.Name, a.Language) + src.ownWhole() src.notPinned(a.Name + " is a package, built from what the registry holds when it is built") reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say) if err != nil { @@ -763,6 +844,7 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactArchive: + src.ownHas(strings.Trim(path.Clean("/"+filepath.ToSlash(a.From)), "/") + "/**") body, err := pack(filepath.Join(tree, a.From)) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) diff --git a/internal/builder/gosource.go b/internal/builder/gosource.go new file mode 100644 index 00000000..c7015887 --- /dev/null +++ b/internal/builder/gosource.go @@ -0,0 +1,485 @@ +package builder + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "go/parser" + "go/token" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// A Go program's build source (novox/hq ADR 0267 rule 1): the files it is built from, derived from its +// import closure rather than listed by hand, because a list drifts from the imports it describes and a +// path missing from it is a real change missed — worse than a needless rebuild. +// +// **The closure read here is never narrower than `go list -deps`.** Every .go file of a package that is +// not a test is read, whatever its build constraint, so the imports are the union over every system and +// tag; a directory is held whole (but for its tests), so a file added to a package is in it; an embed is +// held by the directory its pattern starts in, everything below it. Read with the standard library's +// parser and no toolchain: the build machine carries none, and a closure that needed the network to +// read would be one a build could not say offline. +// +// A build source is a list of entries, relative to the root the build sees: +// +// dir/ a Go package's directory: every file directly in it but its tests (`*_test.go`) +// dir/** everything below a directory (an embed) +// ** the whole tree +// file one file +// +// The root package's directory is `./`. + +// GoPackageDirEntry is the entry for a Go package's directory. +func goPackageDirEntry(dir string) string { + if dir == "" || dir == "." { + return "./" + } + return dir + "/" +} + +// SourceHolds is whether a changed file — a path relative to the root the build source was read in — is +// in that build source. +func SourceHolds(entries []string, file string) bool { + file = strings.TrimPrefix(path.Clean("/"+strings.TrimSpace(file)), "/") + for _, e := range entries { + switch { + case e == "**": + return true + case strings.HasSuffix(e, "/**"): + dir := strings.TrimSuffix(e, "/**") + if dir == "." || dir == "" || file == dir || strings.HasPrefix(file, dir+"/") { + return true + } + case strings.HasSuffix(e, "/"): + dir := strings.TrimSuffix(e, "/") + parent := path.Dir(file) + if (dir == "." && parent == ".") || parent == dir { + if !strings.HasSuffix(file, "_test.go") { + return true + } + } + case e == file: + return true + } + } + return false +} + +// GoBuildSource is the build source of the Go program whose main package is pkg, a directory relative to +// root: the directories of every package of its import closure inside root, the embeds those packages +// name, its module's go.mod, go.sum and vendor/modules.txt, and a go.work wherever one would be read. An +// entry for a file that does not exist is kept: creating it is a change to the build. +// +// Refused — so the build source is not narrowed, and nothing is missed — when the closure cannot be told +// from the files: no go.mod holds the package, a go.work is present, a local replace leaves root, a file +// does not parse, or a cgo preamble reaches outside its directory. +func GoBuildSource(root, pkg string) ([]string, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + rel := path.Clean(strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(pkg)), "/")) + if rel == ".." || strings.HasPrefix(rel, "../") { + return nil, fmt.Errorf("the package %q leaves the tree it is built from", pkg) + } + if info, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil || !info.IsDir() { + return nil, fmt.Errorf("%q is not a directory of the tree it is built from", pkg) + } + // The module holding the package: the nearest go.mod at or above it, within root. + modRoot := "" + for dir := rel; ; dir = path.Dir(dir) { + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.mod")); err == nil { + modRoot = dir + break + } + if dir == "." { + break + } + } + if modRoot == "" { + return nil, fmt.Errorf("no go.mod holds %q within the tree it is built from", pkg) + } + entries := map[string]bool{} + file := func(dir, name string) { + entries[strings.TrimPrefix(path.Join(dir, name), "./")] = true + } + file(modRoot, "go.mod") + file(modRoot, "go.sum") + file(modRoot, "vendor/modules.txt") + // A workspace changes how every import resolves; one present is not read past, one created later is a + // change to the build. + for dir := modRoot; ; dir = path.Dir(dir) { + file(dir, "go.work") + file(dir, "go.work.sum") + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.work")); err == nil { + return nil, fmt.Errorf("%s holds a go.work, and a workspace's imports are not read here", path.Join(dir, "go.work")) + } + if dir == "." { + break + } + } + modPath, replaces, err := readGoMod(filepath.Join(root, filepath.FromSlash(modRoot), "go.mod")) + if err != nil { + return nil, err + } + vendored := false + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(modRoot), "vendor", "modules.txt")); err == nil { + vendored = true + } + // resolve is the directories, relative to root, an import may be read from: none for the standard + // library and for a module outside the tree, which go.mod and go.sum pin. A vendored module replaced + // by a local directory is both — vendor/ under -mod=vendor, the directory under -mod=mod — and both + // are held, so neither way of building it is missed. + resolve := func(importPath string) ([]string, error) { + within := func(prefix, dir string) (string, bool) { + if importPath == prefix { + return dir, true + } + if rest, ok := strings.CutPrefix(importPath, prefix+"/"); ok { + return path.Join(dir, rest), true + } + return "", false + } + if dir, ok := within(modPath, modRoot); ok { + return []string{dir}, nil + } + var dirs []string + for _, r := range replaces { + if sub, ok := within(r.from, ""); ok { + target := path.Clean(path.Join(modRoot, r.to)) + if target == ".." || strings.HasPrefix(target, "../") { + return nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", r.from, r.to) + } + dirs = append(dirs, path.Join(target, sub)) + // Its go.mod states what it requires, read when it is built from there. + entries[path.Join(target, "go.mod")] = true + break + } + } + first, _, _ := strings.Cut(importPath, "/") + if len(dirs) == 0 && !strings.Contains(first, ".") { + return nil, nil // the standard library + } + if vendored { + dirs = append(dirs, path.Join(modRoot, "vendor", importPath)) + } + return dirs, nil + } + + seen := map[string]bool{} + queue := []string{rel} + for len(queue) > 0 { + dir := queue[0] + queue = queue[1:] + if seen[dir] { + continue + } + seen[dir] = true + entries[goPackageDirEntry(dir)] = true + listing, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir))) + if err != nil { + // A package that is not there fails the build that imports it; its directory is held, so + // adding it is a change. + continue + } + for _, f := range listing { + name := f.Name() + if f.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + full := filepath.Join(root, filepath.FromSlash(dir), name) + imports, embeds, err := goFileReads(full) + if err != nil { + return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err) + } + for _, ip := range imports { + targets, err := resolve(ip) + if err != nil { + return nil, err + } + for _, target := range targets { + if !seen[target] { + queue = append(queue, target) + } + } + } + for _, e := range embeds { + entries[path.Join(dir, e)+"/**"] = true + if !strings.ContainsAny(e, "*?[\\") { + entries[path.Join(dir, e)] = true + } + } + } + } + out := make([]string, 0, len(entries)) + for e := range entries { + out = append(out, e) + } + sort.Strings(out) + return out, nil +} + +// goReplace is one local replacement in go.mod: an import path read from a directory. +type goReplace struct{ from, to string } + +// readGoMod is a go.mod's module path and its replacements by a local directory. A replacement by another +// module version is resolved by go.sum, which the build source holds. +func readGoMod(file string) (string, []goReplace, error) { + f, err := os.Open(file) + if err != nil { + return "", nil, err + } + defer f.Close() + var module string + var replaces []goReplace + inReplace := false + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if i := strings.Index(line, "//"); i >= 0 { + line = line[:i] + } + line = strings.TrimSpace(line) + switch { + case line == "": + continue + case inReplace && line == ")": + inReplace = false + continue + case strings.HasPrefix(line, "module "): + module = unquoteGoMod(strings.TrimSpace(strings.TrimPrefix(line, "module"))) + continue + case line == "replace (": + inReplace = true + continue + case strings.HasPrefix(line, "replace "): + line = strings.TrimSpace(strings.TrimPrefix(line, "replace")) + case !inReplace: + continue + } + left, right, found := strings.Cut(line, "=>") + if !found { + continue + } + from := strings.Fields(left) + to := strings.Fields(right) + if len(from) == 0 || len(to) == 0 { + continue + } + target := unquoteGoMod(to[0]) + // A local replacement is a path: ./, ../ or absolute. Anything else names a module version. + if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || target == "." || target == ".." { + replaces = append(replaces, goReplace{from: unquoteGoMod(from[0]), to: target}) + } else if strings.HasPrefix(target, "/") { + return "", nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", from[0], target) + } + } + if err := scanner.Err(); err != nil { + return "", nil, err + } + if module == "" { + return "", nil, fmt.Errorf("%s names no module", file) + } + // The longest replacement first, so a replaced sub-path wins over its parent. + sort.SliceStable(replaces, func(i, j int) bool { return len(replaces[i].from) > len(replaces[j].from) }) + return module, replaces, nil +} + +func unquoteGoMod(s string) string { + if u, err := strconv.Unquote(s); err == nil { + return u + } + return s +} + +// goFileReads is what one Go file makes its build read: the packages it imports, and the directories its +// //go:embed patterns start in, relative to its own directory ("." for the directory itself). +// +// **A cgo preamble reaching outside its directory is refused**: a header included by a relative path, or +// a flag naming ${SRCDIR}/.., is a file of the build no import names. Inside the directory it is held +// already. +func goFileReads(file string) (imports, embeds []string, err error) { + src, err := os.ReadFile(file) + if err != nil { + return nil, nil, err + } + fset := token.NewFileSet() + parsed, err := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.ParseComments) + if err != nil { + return nil, nil, err + } + for _, spec := range parsed.Imports { + ip, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return nil, nil, err + } + if ip == "C" { + // The preamble is the comment before the import; only its #include and #cgo lines read files. + for _, cg := range parsed.Comments { + if cg.End() > spec.Pos() { + continue + } + for _, line := range strings.Split(cg.Text(), "\n") { + line = strings.TrimSpace(line) + if (strings.HasPrefix(line, "#include") || strings.HasPrefix(line, "#cgo")) && strings.Contains(line, "..") { + return nil, nil, errors.New("its cgo preamble names a path outside its directory: " + line) + } + } + } + continue + } + imports = append(imports, ip) + } + // //go:embed directives may stand anywhere in the file, so the whole text is read for them. + scanner := bufio.NewScanner(strings.NewReader(string(src))) + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + rest, ok := strings.CutPrefix(line, "//go:embed") + if !ok || (rest != "" && rest[0] != ' ' && rest[0] != '\t') { + continue + } + patterns, err := embedPatterns(rest) + if err != nil { + return nil, nil, err + } + for _, p := range patterns { + embeds = append(embeds, embedRoot(p)) + } + } + return imports, embeds, scanner.Err() +} + +// embedPatterns splits a //go:embed line's patterns: separated by spaces, each possibly quoted. +func embedPatterns(s string) ([]string, error) { + var out []string + s = strings.TrimSpace(s) + for s != "" { + var p string + switch s[0] { + case '"', '`': + q, err := strconv.QuotedPrefix(s) + if err != nil { + return nil, fmt.Errorf("an embed pattern does not parse: %w", err) + } + if p, err = strconv.Unquote(q); err != nil { + return nil, err + } + s = s[len(q):] + default: + end := strings.IndexAny(s, " \t") + if end < 0 { + end = len(s) + } + p, s = s[:end], s[end:] + } + out = append(out, p) + s = strings.TrimSpace(s) + } + return out, nil +} + +// embedRoot is the directory an embed pattern starts in, relative to the package: its leading elements +// without a wildcard. A pattern naming a file or a directory outright is held as itself. +func embedRoot(pattern string) string { + pattern = strings.TrimPrefix(pattern, "all:") + var kept []string + for _, el := range strings.Split(pattern, "/") { + if strings.ContainsAny(el, "*?[\\") { + break + } + kept = append(kept, el) + } + if len(kept) == 0 { + return "." + } + return path.Clean(strings.Join(kept, "/")) +} + +// narrowTree copies into dst the files of src a build source holds, and nothing else — never `.git` — and +// returns a fingerprint of what it copied: each file's path, mode and content, hashed in path order. **A +// build handed only its build source cannot read past it** (novox/hq ADR 0267 rule 3): a recipe that +// copies a file outside it fails, naming the file, where it would have built and been missed. +func narrowTree(src, dst string, entries []string) (string, error) { + if err := os.RemoveAll(dst); err != nil { + return "", err + } + if err := os.MkdirAll(dst, 0o755); err != nil { + return "", err + } + var lines []string + err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + if !SourceHolds(entries, rel) { + return nil + } + out := filepath.Join(dst, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return err + } + info, err := d.Info() + if err != nil { + return err + } + if info.Mode()&fs.ModeSymlink != 0 { + // A link in the repository is copied as the link it is, and what it names said in the + // fingerprint. + target, err := os.Readlink(p) + if err != nil { + return err + } + lines = append(lines, fmt.Sprintf("%s link %s", rel, target)) + return os.Symlink(target, out) + } + if !info.Mode().IsRegular() { + return nil + } + in, err := os.Open(p) + if err != nil { + return err + } + defer in.Close() + w, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + sum := sha256.New() + if _, err := io.Copy(io.MultiWriter(w, sum), in); err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + lines = append(lines, fmt.Sprintf("%s %o %s", rel, info.Mode().Perm()&0o111, hex.EncodeToString(sum.Sum(nil)))) + return nil + }) + if err != nil { + return "", err + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return "narrow:" + hex.EncodeToString(sum[:]), nil +} diff --git a/internal/builder/gosource_test.go b/internal/builder/gosource_test.go new file mode 100644 index 00000000..d51394fd --- /dev/null +++ b/internal/builder/gosource_test.go @@ -0,0 +1,288 @@ +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// A Go program's build source is its import closure (novox/hq ADR 0267 rule 1): never narrower than what +// `go build` reads, whatever the system, the tags, the vendoring or the embeds. + +// aGoTree writes files under a fresh directory and returns it. +func aGoTree(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for name, body := range files { + full := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +// aProgram is a module whose program imports its own packages, a vendored module, a local replacement +// that is vendored too, an embed and a package only one system builds; beside them, a package nothing +// imports and one only a test imports. +var aProgram = map[string]string{ + "go.mod": "module example.com/fix\n\ngo 1.22\n\nrequire (\n\texample.org/dep v1.0.0\n\texample.net/local v0.0.0\n)\n\n" + + "replace example.net/local => ./third_party/local\n", + "go.sum": "", + "vendor/modules.txt": "# example.net/local v0.0.0 => ./third_party/local\n## explicit; go 1.22\nexample.net/local/pkg\n" + + "# example.org/dep v1.0.0\n## explicit; go 1.22\nexample.org/dep\nexample.org/dep/sub\n# example.net/local => ./third_party/local\n", + "vendor/example.org/dep/dep.go": "package dep\n\nimport _ \"example.org/dep/sub\"\n", + "vendor/example.org/dep/sub/sub.go": "package sub\n", + "vendor/example.org/other/other.go": "package other\n", + "vendor/example.net/local/pkg/p.go": "package pkg\n", + "third_party/local/go.mod": "module example.net/local\n\ngo 1.22\n", + "third_party/local/pkg/p.go": "package pkg\n", + "cmd/prog/main.go": "package main\n\nimport (\n\t\"fmt\"\n\n\t_ \"example.com/fix/emb\"\n\t\"example.com/fix/lib\"\n" + + "\t_ \"example.net/local/pkg\"\n\t_ \"example.org/dep\"\n)\n\nfunc main() { fmt.Println(lib.X) }\n", + "lib/lib.go": "package lib\n\nconst X = 1\n", + "lib/lib_plan9.go": "//go:build plan9\n\npackage lib\n\nimport _ \"example.com/fix/plan9only\"\n", + "lib/lib_test.go": "package lib\n\nimport _ \"example.com/fix/testonly\"\n", + "lib/lib_amd64.s": "", + "lib/sub/sub.go": "package sub\n", + "plan9only/p.go": "package plan9only\n", + "testonly/t.go": "package testonly\n", + "unrelated/u.go": "package unrelated\n", + "emb/emb.go": "package emb\n\nimport \"embed\"\n\n//go:embed static/*\nvar Static embed.FS\n\n" + + "//go:embed \"a b.txt\"\nvar Text string\n", + "emb/static/index.html": "x", + "emb/static/deep/style.css": "x", + "emb/a b.txt": "x", + "README.md": "x", +} + +func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) { + root := aGoTree(t, aProgram) + got, err := GoBuildSource(root, "cmd/prog") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + file string + held bool + why string + }{ + {"cmd/prog/main.go", true, "the program itself"}, + {"cmd/prog/helper.go", true, "a file added to the program's package"}, + {"lib/lib.go", true, "a package it imports"}, + {"lib/lib_amd64.s", true, "assembly beside a package's Go"}, + {"lib/lib_plan9.go", true, "a file one system builds"}, + {"plan9only/p.go", true, "what a file one system builds imports"}, + {"emb/static/index.html", true, "an embedded file"}, + {"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"}, + {"emb/a b.txt", true, "an embed named outright, quoted"}, + {"vendor/example.org/dep/dep.go", true, "a vendored package it imports"}, + {"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"}, + {"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"}, + {"third_party/local/pkg/p.go", true, "a replaced module, at its directory"}, + {"third_party/local/go.mod", true, "a replaced module's requirements"}, + {"go.mod", true, "the module's requirements"}, + {"go.sum", true, "the module's sums"}, + {"vendor/modules.txt", true, "the vendored modules"}, + {"go.work", true, "a workspace, were one made"}, + {"lib/lib_test.go", false, "a test is not built"}, + {"testonly/t.go", false, "a package only a test imports"}, + {"lib/sub/sub.go", false, "a package below an imported one, not imported"}, + {"unrelated/u.go", false, "a package nothing imports"}, + {"vendor/example.org/other/other.go", false, "a vendored package nothing imports"}, + {"README.md", false, "a file no build reads"}, + } { + if SourceHolds(got, c.file) != c.held { + t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got) + } + } + + // **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds, + // is held. Where no go command is at hand, said, not passed. + goCmd, err := exec.LookPath("go") + if err != nil { + t.Skip("NOT COMPARED: no go command to list the closure with") + } + list := exec.Command(goCmd, "list", "-deps", "-f", + `{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}} +{{end}}{{range .SFiles}}{{$d}}/{{.}} +{{end}}{{range .EmbedFiles}}{{$d}}/{{.}} +{{end}}{{end}}`, "./cmd/prog") + list.Dir = root + list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64") + out, err := list.CombinedOutput() + if err != nil { + t.Fatalf("go list: %v\n%s", err, out) + } + real, _ := filepath.EvalSymlinks(root) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + rel, err := filepath.Rel(real, line) + if err != nil || strings.HasPrefix(rel, "..") { + rel, _ = filepath.Rel(root, line) + } + if !SourceHolds(got, filepath.ToSlash(rel)) { + t.Errorf("go list builds %s, and the build source does not hold it", rel) + } + } +} + +// A file added to the program's import closure moves its build source with it: the closure read again +// grows by the package. +func TestTheBuildSourceGrowsWithAnImport(t *testing.T) { + files := map[string]string{} + for k, v := range aProgram { + files[k] = v + } + before, err := GoBuildSource(aGoTree(t, files), "cmd/prog") + if err != nil { + t.Fatal(err) + } + if SourceHolds(before, "unrelated/u.go") { + t.Fatal("held before it was imported") + } + files["cmd/prog/more.go"] = "package main\n\nimport _ \"example.com/fix/unrelated\"\n" + after, err := GoBuildSource(aGoTree(t, files), "cmd/prog") + if err != nil { + t.Fatal(err) + } + if !SourceHolds(after, "unrelated/u.go") { + t.Fatalf("an import added did not grow the build source: %v", after) + } +} + +// What cannot be read is refused, so the build source is not narrowed and nothing is missed. +func TestABuildSourceThatCannotBeReadIsRefused(t *testing.T) { + for _, c := range []struct { + what string + files map[string]string + pkg string + says string + }{ + {"no go.mod", map[string]string{"cmd/p/main.go": "package main\n"}, "cmd/p", "no go.mod"}, + {"a workspace", map[string]string{"go.mod": "module x\n", "go.work": "go 1.22\n", "p/main.go": "package main\n"}, + "p", "go.work"}, + {"a local replacement outside the tree", map[string]string{ + "go.mod": "module x\n\nreplace y => ../y\n", "p/main.go": "package main\n\nimport _ \"y\"\n"}, "p", "outside"}, + {"a cgo header outside the directory", map[string]string{ + "go.mod": "module x\n", "p/main.go": "package main\n\n// #include \"../h/h.h\"\nimport \"C\"\n"}, "p", "cgo"}, + {"a file that does not parse", map[string]string{"go.mod": "module x\n", "p/main.go": "package main\n\nimport (\n"}, + "p", "main.go"}, + {"a package that leaves the tree", map[string]string{"go.mod": "module x\n"}, "../elsewhere", "leaves"}, + } { + _, err := GoBuildSource(aGoTree(t, c.files), c.pkg) + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v, wanted a refusal saying %q", c.what, err, c.says) + } + } + // A cgo header in the package's own directory is held already, and is no refusal. + if _, err := GoBuildSource(aGoTree(t, map[string]string{"go.mod": "module x\n", + "p/main.go": "package main\n\n// #include \"h.h\"\nimport \"C\"\n"}), "p"); err != nil { + t.Errorf("a header in the package's directory was refused: %v", err) + } +} + +func TestABuildSourceHoldsWhatItsEntriesSay(t *testing.T) { + entries := []string{"./", "cmd/p/", "web/**", "go.mod"} + for file, want := range map[string]bool{ + "main.go": true, "main_test.go": false, "cmd/p/x.go": true, "cmd/p/x_test.go": false, "cmd/p/sub/y.go": false, + "cmd/px/x.go": false, "web/a/b/c.css": true, "web": true, "webx/a": false, "go.mod": true, "docs/x.md": false, + "/cmd/p/x.go": true, "cmd/p/../q/x.go": false, + } { + if SourceHolds(entries, file) != want { + t.Errorf("%s: held %v, wanted %v", file, !want, want) + } + } + if !SourceHolds([]string{"**"}, "anything/at/all") { + t.Error("the whole tree does not hold a file") + } +} + +// **A build handed its build source reads nothing else** (rule 3): the narrowed tree holds the source's +// files and no others — never .git — and its fingerprint changes with them and only with them. +func TestANarrowedTreeHoldsTheBuildSourceAndNothingElse(t *testing.T) { + files := map[string]string{} + for k, v := range aProgram { + files[k] = v + } + files[".git/HEAD"] = "ref: refs/heads/main\n" + src := aGoTree(t, files) + entries, err := GoBuildSource(src, "cmd/prog") + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(t.TempDir(), "narrow") + one, err := narrowTree(src, dst, entries) + if err != nil { + t.Fatal(err) + } + for file, want := range map[string]bool{"cmd/prog/main.go": true, "lib/lib.go": true, "emb/static/deep/style.css": true, + "lib/lib_test.go": false, "unrelated/u.go": false, "README.md": false, ".git/HEAD": false} { + _, err := os.Stat(filepath.Join(dst, filepath.FromSlash(file))) + if (err == nil) != want { + t.Errorf("%s: copied %v, wanted %v", file, err == nil, want) + } + } + // Outside the build source: one fingerprint. + files["README.md"] = "changed" + files["unrelated/u.go"] = "package unrelated\n\nconst Changed = 1\n" + again, err := narrowTree(aGoTree(t, files), filepath.Join(t.TempDir(), "n"), entries) + if err != nil || again != one { + t.Fatalf("a change outside the build source changed its fingerprint: %s %s %v", one, again, err) + } + // Inside it: another. + files["lib/lib.go"] = "package lib\n\nconst X = 2\n" + moved, err := narrowTree(aGoTree(t, files), filepath.Join(t.TempDir(), "n"), entries) + if err != nil || moved == one { + t.Fatalf("a change inside the build source kept its fingerprint: %s %v", moved, err) + } +} + +// This repository's own programs: the route proxy's build source is not the controller's, and neither +// holds the other's command. +func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) { + root := filepath.Join("..", "..") + proxy, err := GoBuildSource(root, "examples/route-proxy") + if err != nil { + t.Fatal(err) + } + controller, err := GoBuildSource(root, "cmd/mesh-controller") + if err != nil { + t.Fatal(err) + } + builder, err := GoBuildSource(root, "cmd/mesh-builder") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + entries []string + name string + file string + held bool + }{ + {proxy, "the route proxy", "examples/route-proxy/main.go", true}, + {proxy, "the route proxy", "internal/broker/broker.go", true}, + {proxy, "the route proxy", "cmd/mesh-controller/main.go", false}, + {proxy, "the route proxy", "internal/conditions/condition.go", false}, + {proxy, "the route proxy", "README.md", false}, + {controller, "the controller", "cmd/mesh-controller/main.go", true}, + {controller, "the controller", "internal/conditions/condition.go", true}, + {controller, "the controller", "internal/inventory/migrations/0001-nodes.sql", true}, + {controller, "the controller", "examples/route-proxy/main.go", false}, + {controller, "the controller", "cmd/mesh-builder/main.go", false}, + {controller, "the controller", "README.md", false}, + {builder, "the build seat's program", "cmd/mesh-builder/main.go", true}, + {builder, "the build seat's program", "internal/conditions/condition.go", false}, + {builder, "the build seat's program", "cmd/mesh-controller/main.go", false}, + } { + if SourceHolds(c.entries, c.file) != c.held { + t.Errorf("%s: %s held %v, wanted %v", c.name, c.file, !c.held, c.held) + } + } +} diff --git a/internal/builder/source.go b/internal/builder/source.go index a6aeb2e5..a5f54beb 100644 --- a/internal/builder/source.go +++ b/internal/builder/source.go @@ -9,6 +9,8 @@ import ( "path/filepath" "sort" "strings" + + "github.com/novox/mesh-controller/internal/catalogue" ) // A build's source fingerprint: what it was made from, hashed (novox/hq issue 280). @@ -49,10 +51,132 @@ type sourceInputs struct { toolchains map[string]string // unpinned is why this build has no fingerprint: empty when it has one. unpinned string + + // prefix is the module's directory within its repository, empty at the root. + prefix string + // own is the module's build source in its own repository, relative to the module's directory, and + // whole when an artifact's is not known (novox/hq ADR 0267). + own map[string]bool + ownIsAll bool + // read is, per context (repository and ref), the build source read there; nil for one read whole. + read map[string]map[string]bool + readAs map[string]catalogue.ArtifactContext } func newSourceInputs(module string) *sourceInputs { - return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}} + return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}, + own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{}} +} + +// ownHas adds entries, relative to the module's directory, to its build source in its own repository. +func (s *sourceInputs) ownHas(entries ...string) { + if s == nil { + return + } + for _, e := range entries { + s.own[e] = true + } +} + +// ownWhole says an artifact's build source in the module's own repository is not known: the module's +// whole directory is its source, as it was before. +func (s *sourceInputs) ownWhole() { + if s != nil { + s.ownIsAll = true + } +} + +func contextKey(c catalogue.ArtifactContext) string { return c.Repository + "#" + c.Ref } + +// readIn adds entries to the build source read in a context; one read whole stays whole. +func (s *sourceInputs) readIn(c catalogue.ArtifactContext, entries []string) { + if s == nil { + return + } + key := contextKey(c) + s.readAs[key] = c + set, known := s.read[key] + if known && set == nil { + return + } + if set == nil { + set = map[string]bool{} + s.read[key] = set + } + for _, e := range entries { + set[e] = true + } +} + +// readWhole says a context is read whole by an artifact. +func (s *sourceInputs) readWhole(c catalogue.ArtifactContext) { + if s == nil { + return + } + key := contextKey(c) + s.readAs[key] = c + s.read[key] = nil +} + +// buildSources is what the build says it was made from, per repository: its own (module.json always, +// and every artifact's) unless an artifact's is not known, and each context not read whole. +func (s *sourceInputs) buildSources() []BuildSource { + if s == nil { + return nil + } + var out []BuildSource + if !s.ownIsAll { + own := []string{withPrefix(s.prefix, ManifestName)} + for e := range s.own { + own = append(own, withPrefix(s.prefix, e)) + } + sort.Strings(own) + out = append(out, BuildSource{Paths: compactSorted(own)}) + } + var keys []string + for k := range s.read { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + set := s.read[k] + if set == nil { + continue + } + var paths []string + for e := range set { + paths = append(paths, e) + } + sort.Strings(paths) + c := s.readAs[k] + out = append(out, BuildSource{Repository: c.Repository, Ref: c.Ref, Paths: paths}) + } + return out +} + +// withPrefix is an entry relative to the module's directory made relative to its repository's root. +func withPrefix(prefix, entry string) string { + entry = strings.TrimPrefix(entry, "./") + if prefix == "" { + if entry == "" { + return "./" + } + return entry + } + if entry == "" { + return prefix + "/" + } + return prefix + "/" + entry +} + +func compactSorted(in []string) []string { + var out []string + for i, e := range in { + if i == 0 || e != in[i-1] { + out = append(out, e) + } + } + return out } // notPinned marks the build as one its source does not pin; the first reason stands. diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 23067e2f..21f0e296 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -322,6 +322,17 @@ func (b *Build) problems(module string) []string { "everything else brings its own recipe", module, a.Name, a.Kind)) } } + if a.Compiles != "" { + if a.Kind != ArtifactImage { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+ + "told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind)) + } else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." || + strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") { + problems = append(problems, fmt.Sprintf( + "%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles)) + } + } // An upstream image is named, not read from the repository, so the path rule does not // apply to it — and applying it anyway would refuse every reference with a registry host // in it. diff --git a/internal/catalogue/build_test.go b/internal/catalogue/build_test.go index 34a86944..c17c0ebe 100644 --- a/internal/catalogue/build_test.go +++ b/internal/catalogue/build_test.go @@ -180,3 +180,24 @@ func TestAResourceNamingAPackageIsRefused(t *testing.T) { t.Fatal("a resource backed by a package was accepted; a package is not a resource") } } + +// An image names the Go package its recipe compiles (novox/hq ADR 0267): within the tree it is built in, +// and only an image says one. +func TestOnlyAnImageNamesThePackageItCompilesWithinItsTree(t *testing.T) { + m, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"image","from":"Dockerfile","compiles":"cmd/x"}]}}`)) + if err != nil || m.Build.Artifacts[0].Compiles != "cmd/x" { + t.Fatalf("an image naming its package was refused or lost it: %v", err) + } + for _, c := range []struct{ artifact, says string }{ + {`{"name":"x","kind":"archive","from":"files","compiles":"cmd/x"}`, "Only an image"}, + {`{"name":"x","kind":"image","from":"Dockerfile","compiles":"../x"}`, "leaves the tree"}, + {`{"name":"x","kind":"image","from":"Dockerfile","compiles":"/x"}`, "leaves the tree"}, + {`{"name":"x","kind":"image","from":"Dockerfile","compiles":"a/../../x"}`, "leaves the tree"}, + } { + _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[` + c.artifact + `]}}`)) + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v, wanted a refusal saying %q", c.artifact, err, c.says) + } + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 842580cf..56c391bf 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -905,6 +905,16 @@ type Artifact struct { // image built from this same module's own repository, the same as every other artifact. Context *ArtifactContext `json:"context,omitempty"` + // Compiles is the Go package an image's recipe compiles, relative to the tree it is built in — its + // context, or the module's directory (novox/hq ADR 0267 rule 1). + // + // **What a build is made from is derived, never listed.** The build seat reads the package's import + // closure and hands the recipe that and nothing else, so a merge elsewhere in a shared repository is + // no change to this image, and a recipe copying a file outside the closure fails by name instead of + // building from something no merge is mapped onto. Empty for an image that compiles no Go: its whole + // tree is its build source, as before. + Compiles string `json:"compiles,omitempty"` + // System is the operating system this artifact is compiled for, for a bundle whose output is a // binary rather than portable code (novox/hq ADR 0142). // diff --git a/internal/link/build.go b/internal/link/build.go index 9cb65fa5..aa90165e 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -224,6 +224,12 @@ type BuildResult struct { // manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131). Read []ReadRepository `json:"read,omitempty"` + // Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): the + // entries the planner maps the next merge's changed files onto. Said only for a build of a commit on + // the trunk, and only for a repository whose every artifact's build source the builder knows; a + // repository it says nothing of is read whole, as before. Empty from a builder that predates it. + Sources []BuildSource `json:"sources,omitempty"` + // Trunk is the repository's default branch at the build, and OnTrunk whether the commit built is on it // (novox/hq ADR 0238): **only a commit on the trunk is published** — the controller refuses to register // a build of one off it. Empty Trunk is a build seat that could not say, or predates the rule. @@ -263,6 +269,16 @@ type ReadRepository struct { Ref string `json:"ref,omitempty"` } +// BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref +// a context's, as its manifest names it, and empty for the module's own; Paths the entries, relative to +// that repository's root — `dir/` a Go package's directory but its tests, `dir/**` everything below a +// directory, `**` the whole tree, anything else one file (builder.SourceHolds). +type BuildSource struct { + Repository string `json:"repository,omitempty"` + Ref string `json:"ref,omitempty"` + Paths []string `json:"paths"` +} + // MadeArtifact is one thing a build produced, as a person would want it reported. type MadeArtifact struct { Name string `json:"name"`