Merge pull request 'A build says its build source; an image compiling Go is handed only that (hq ADR 0267, issue 363)' (#192) from fix/0267-a-build-says-its-build-source into main
This commit was merged in pull request #192.
This commit is contained in:
@@ -306,6 +306,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
for _, r := range built.Read {
|
for _, r := range built.Read {
|
||||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
}
|
}
|
||||||
|
for _, s := range built.Sources {
|
||||||
|
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||||
|
}
|
||||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,202 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a build says it was made from, as files (novox/hq ADR 0267), and what a build compiling a Go
|
||||||
|
// program is handed.
|
||||||
|
|
||||||
|
// onTrunk answers the trunk's questions as a clone of a commit on main would, or off it.
|
||||||
|
type onTrunk struct {
|
||||||
|
*recorded
|
||||||
|
off bool
|
||||||
|
// behind is a commit on the trunk that is not its head: an older commit built by hand.
|
||||||
|
behind bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o onTrunk) run(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||||
|
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
|
||||||
|
return "origin/main\n", nil
|
||||||
|
}
|
||||||
|
if name == "git" && len(args) > 1 && args[0] == "rev-parse" && args[1] == "origin/main" && o.behind {
|
||||||
|
return "feedfacefeedfacefeedfacefeedfacefeedface\n", nil
|
||||||
|
}
|
||||||
|
if name == "git" && len(args) > 0 && args[0] == "merge-base" && o.off {
|
||||||
|
return "", os.ErrNotExist
|
||||||
|
}
|
||||||
|
return compiling{o.recorded}.run(ctx, dir, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// aSharedRepository is a repository holding two programs that share a package, as the controller's does.
|
||||||
|
func aSharedRepository(readme, shared string) map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
"go.mod": "module example.com/ctl\n\ngo 1.22\n",
|
||||||
|
"go.sum": "",
|
||||||
|
"README.md": readme,
|
||||||
|
"cmd/ctl/main.go": "package main\n\nimport _ \"example.com/ctl/internal/shared\"\n\nfunc main() {}\n",
|
||||||
|
"proxy/main.go": "package main\n\nimport _ \"example.com/ctl/internal/shared\"\n\nfunc main() {}\n",
|
||||||
|
"internal/shared/s.go": "package shared\n\nconst S = " + shared + "\n",
|
||||||
|
"internal/only/o.go": "package only\n",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const aProxy = `{"module":"route-proxy","version":"1",
|
||||||
|
"build":{"artifacts":[
|
||||||
|
{"name":"server","kind":"image","from":"Dockerfile","compiles":"proxy",
|
||||||
|
"context":{"repository":"https://forge.invalid/ctl.git","ref":"main"}},
|
||||||
|
{"name":"trust","kind":"upstream","from":"alpine@sha256:` + "3333333333333333333333333333333333333333333333333333333333333333" + `"}]}}`
|
||||||
|
|
||||||
|
func buildTheProxy(t *testing.T, context_ map[string]string, off bool, behind ...bool) (Result, *recorded, string) {
|
||||||
|
t.Helper()
|
||||||
|
r := &recorded{
|
||||||
|
contents: map[string]string{"modules/route-proxy/" + ManifestName: aProxy, "modules/route-proxy/Dockerfile": "FROM scratch\nCOPY . .\n", "modules/route-proxy/README.md": "x"},
|
||||||
|
secondary: map[string]map[string]string{"https://forge.invalid/ctl.git": context_},
|
||||||
|
}
|
||||||
|
workspace := t.TempDir()
|
||||||
|
got, err := Build(context.Background(), onTrunk{r, off, len(behind) > 0 && behind[0]}.run, r,
|
||||||
|
"https://forge.invalid/catalogue.git", "modules/route-proxy", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return got, r, workspace
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnImageCompilingGoIsHandedItsBuildSourceAndSaysIt(t *testing.T) {
|
||||||
|
got, r, workspace := buildTheProxy(t, aSharedRepository("one", "1"), false)
|
||||||
|
|
||||||
|
// Built in the narrowed tree, which holds the program's closure and nothing else.
|
||||||
|
at := ""
|
||||||
|
for i, line := range r.ran {
|
||||||
|
if strings.HasPrefix(line, "docker build ") {
|
||||||
|
at = r.dirs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filepath.Base(at) != "narrow-server" {
|
||||||
|
t.Fatalf("docker build ran in %q, not the narrowed build source", at)
|
||||||
|
}
|
||||||
|
for file, want := range map[string]bool{"proxy/main.go": true, "internal/shared/s.go": true, "go.mod": true,
|
||||||
|
"cmd/ctl/main.go": false, "internal/only/o.go": false, "README.md": false} {
|
||||||
|
_, err := os.Stat(filepath.Join(workspace, "narrow-server", filepath.FromSlash(file)))
|
||||||
|
if (err == nil) != want {
|
||||||
|
t.Errorf("%s handed to the recipe: %v, wanted %v", file, err == nil, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Said per repository: its own, the manifest and the recipe; the context's, the closure.
|
||||||
|
if len(got.Sources) != 2 {
|
||||||
|
t.Fatalf("sources %+v", got.Sources)
|
||||||
|
}
|
||||||
|
own, ctx := got.Sources[0], got.Sources[1]
|
||||||
|
if own.Repository != "" || !slices.Equal(own.Paths, []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}) {
|
||||||
|
t.Errorf("its own build source: %+v", own)
|
||||||
|
}
|
||||||
|
if ctx.Repository != "https://forge.invalid/ctl.git" || ctx.Ref != "main" {
|
||||||
|
t.Errorf("the context's build source names %q at %q", ctx.Repository, ctx.Ref)
|
||||||
|
}
|
||||||
|
for file, want := range map[string]bool{"proxy/main.go": true, "internal/shared/s.go": true, "go.mod": true,
|
||||||
|
"cmd/ctl/main.go": false, "README.md": false} {
|
||||||
|
if SourceHolds(ctx.Paths, file) != want {
|
||||||
|
t.Errorf("the context's build source holds %s: %v, wanted %v (%v)", file, !want, want, ctx.Paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The fingerprint is over the build source** (rule 5): a change outside it is one build, inside it another.
|
||||||
|
readme, _, _ := buildTheProxy(t, aSharedRepository("two", "1"), false)
|
||||||
|
if readme.Source != got.Source {
|
||||||
|
t.Errorf("a README of the context changed the fingerprint: %s %s", got.Source, readme.Source)
|
||||||
|
}
|
||||||
|
shared, _, _ := buildTheProxy(t, aSharedRepository("one", "2"), false)
|
||||||
|
if shared.Source == got.Source {
|
||||||
|
t.Error("a change to the program's closure kept its fingerprint")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build off the trunk, or of a trunk commit that is not its head, says no build source: the planner maps
|
||||||
|
// a merge onto the trunk head's, and an older commit's closure lacks what was imported since.
|
||||||
|
func TestABuildOffTheTrunksHeadSaysNoBuildSource(t *testing.T) {
|
||||||
|
got, _, _ := buildTheProxy(t, aSharedRepository("one", "1"), true)
|
||||||
|
if len(got.Sources) != 0 {
|
||||||
|
t.Fatalf("a build off the trunk said %+v", got.Sources)
|
||||||
|
}
|
||||||
|
got, _, _ = buildTheProxy(t, aSharedRepository("one", "1"), false, true)
|
||||||
|
if len(got.Sources) != 0 {
|
||||||
|
t.Fatalf("a build of an older trunk commit said %+v", got.Sources)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An archive of the module's whole directory holds every file of it.
|
||||||
|
func TestAnArchiveOfTheWholeDirectoryHoldsIt(t *testing.T) {
|
||||||
|
manifest := `{"module":"look","version":"1","build":{"artifacts":[{"name":"all","kind":"archive","from":"."}]},
|
||||||
|
"resources":[{"id":"files","type":"archive","path":"/opt/look","artifact":"all"}]}`
|
||||||
|
r := &recorded{contents: map[string]string{"modules/look/" + ManifestName: manifest, "modules/look/a/b.css": "x"}}
|
||||||
|
got, err := Build(context.Background(), onTrunk{recorded: r}.run, r,
|
||||||
|
"https://forge.invalid/catalogue.git", "modules/look", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Sources) != 1 || !SourceHolds(got.Sources[0].Paths, "modules/look/a/b.css") ||
|
||||||
|
SourceHolds(got.Sources[0].Paths, "modules/other/x") {
|
||||||
|
t.Fatalf("sources %+v", got.Sources)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A recipe reading past its build source fails, naming what it could not find — in the real docker build;
|
||||||
|
// here, the file is simply not in the tree it is handed, which is what makes that so.
|
||||||
|
func TestAnImageCompilingANonexistentPackageFails(t *testing.T) {
|
||||||
|
r := &recorded{
|
||||||
|
contents: map[string]string{ManifestName: strings.Replace(aProxy, `"compiles":"proxy"`, `"compiles":"nowhere"`, 1),
|
||||||
|
"Dockerfile": "FROM scratch\n"},
|
||||||
|
secondary: map[string]map[string]string{"https://forge.invalid/ctl.git": aSharedRepository("one", "1")},
|
||||||
|
}
|
||||||
|
_, err := Build(context.Background(), onTrunk{recorded: r}.run, r,
|
||||||
|
"https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "nowhere") {
|
||||||
|
t.Fatalf("a package that is not there built: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Go bundle of a module built from its repository's root says its import closure, and the manifest;
|
||||||
|
// an image that compiles nothing it was told of leaves the module's source whole, and says none.
|
||||||
|
func TestAGoBundleSaysItsClosureAndAnUntoldImageNothing(t *testing.T) {
|
||||||
|
files := aSharedRepository("one", "1")
|
||||||
|
manifest := `{"module":"ctl","version":"1","build":{"artifacts":[
|
||||||
|
{"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},
|
||||||
|
"resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}`
|
||||||
|
r := &recorded{contents: map[string]string{ManifestName: manifest}}
|
||||||
|
for k, v := range files {
|
||||||
|
r.contents[k] = v
|
||||||
|
}
|
||||||
|
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
|
||||||
|
got, err := Build(context.Background(), onTrunk{recorded: r}.run, r,
|
||||||
|
"https://forge.invalid/ctl.git", "", "", t.TempDir(), held, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Sources) != 1 || got.Sources[0].Repository != "" {
|
||||||
|
t.Fatalf("sources %+v", got.Sources)
|
||||||
|
}
|
||||||
|
for file, want := range map[string]bool{"cmd/ctl/main.go": true, "internal/shared/s.go": true, ManifestName: true,
|
||||||
|
"go.sum": true, "proxy/main.go": false, "README.md": false, "internal/only/o.go": false} {
|
||||||
|
if SourceHolds(got.Sources[0].Paths, file) != want {
|
||||||
|
t.Errorf("%s: held %v, wanted %v (%v)", file, !want, want, got.Sources[0].Paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
untold := `{"module":"ctl","version":"1","build":{"artifacts":[
|
||||||
|
{"name":"server","kind":"image","from":"Dockerfile"}]}}`
|
||||||
|
r = &recorded{contents: map[string]string{ManifestName: untold, "Dockerfile": "FROM scratch\n"}}
|
||||||
|
got, err = Build(context.Background(), onTrunk{recorded: r}.run, r,
|
||||||
|
"https://forge.invalid/ctl.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Sources) != 0 {
|
||||||
|
t.Fatalf("an image compiling nothing it was told of said a build source: %+v", got.Sources)
|
||||||
|
}
|
||||||
|
}
|
||||||
+113
-5
@@ -11,6 +11,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -87,6 +88,23 @@ type Result struct {
|
|||||||
// pin the build. Two builds with one fingerprint are one build, whatever digests they made
|
// pin the build. Two builds with one fingerprint are one build, whatever digests they made
|
||||||
// (novox/hq issue 280).
|
// (novox/hq issue 280).
|
||||||
Source string
|
Source string
|
||||||
|
|
||||||
|
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
|
||||||
|
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
|
||||||
|
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
|
||||||
|
// artifact's build source is known — a Go program's import closure, an archive's directory, an image's
|
||||||
|
// recipe and the package it compiles; for any other, nothing is said and the whole of what the build
|
||||||
|
// sees stays its source, as before.
|
||||||
|
Sources []BuildSource
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267).
|
||||||
|
type BuildSource struct {
|
||||||
|
// Repository and Ref are a context's, as the manifest names it; empty for the module's own.
|
||||||
|
Repository string
|
||||||
|
Ref string
|
||||||
|
// Paths are the entries, relative to the repository's root (SourceHolds).
|
||||||
|
Paths []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// GitCredential is the forge credential a clone may present when the server asks for one.
|
// GitCredential is the forge credential a clone may present when the server asks for one.
|
||||||
@@ -206,6 +224,10 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
|
|
||||||
// What it is made from, for its source fingerprint: the module's own tree first.
|
// What it is made from, for its source fingerprint: the module's own tree first.
|
||||||
src := newSourceInputs(manifest.Module)
|
src := newSourceInputs(manifest.Module)
|
||||||
|
src.prefix = strings.Trim(filepath.ToSlash(filepath.Clean(path)), "/")
|
||||||
|
if src.prefix == "." {
|
||||||
|
src.prefix = ""
|
||||||
|
}
|
||||||
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
|
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
|
||||||
src.notPinned("its tree could not be named: " + err.Error())
|
src.notPinned("its tree could not be named: " + err.Error())
|
||||||
}
|
}
|
||||||
@@ -298,9 +320,23 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
if fingerprint == "" {
|
if fingerprint == "" {
|
||||||
say("source", "no source fingerprint: %s", orNoTree(src.unpinned))
|
say("source", "no source fingerprint: %s", orNoTree(src.unpinned))
|
||||||
}
|
}
|
||||||
|
// **Only a build of the trunk's head says its build source** (novox/hq ADR 0267): the planner maps the
|
||||||
|
// next merge onto the build source of the newest build, and a branch's closure — or an older trunk
|
||||||
|
// commit's, built by hand — is not the trunk's. Nor does a build whose context was not its trunk's head.
|
||||||
|
var sources []BuildSource
|
||||||
|
if trunk != "" && onTrunk && src.contextsAtHead && atTrunkHead(ctx, run, tree, commit, trunk) {
|
||||||
|
sources = src.buildSources()
|
||||||
|
for _, s := range sources {
|
||||||
|
where := "its own repository"
|
||||||
|
if s.Repository != "" {
|
||||||
|
where = s.Repository
|
||||||
|
}
|
||||||
|
say("source", "%d path(s) of %s", len(s.Paths), where)
|
||||||
|
}
|
||||||
|
}
|
||||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
||||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches}, nil
|
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
||||||
@@ -345,6 +381,28 @@ func trunkOf(ctx context.Context, run Runner, clone, commit string) (string, boo
|
|||||||
return trunk, err == nil
|
return trunk, err == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// atTrunkHead is whether a clone's commit is its trunk's head as the clone holds it.
|
||||||
|
func atTrunkHead(ctx context.Context, run Runner, clone, commit, trunk string) bool {
|
||||||
|
head, err := run(ctx, clone, "git", "rev-parse", "origin/"+trunk)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
at, err := run(ctx, clone, "git", "rev-parse", commit)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(head) != "" && strings.TrimSpace(head) == strings.TrimSpace(at)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanEntry is a path of the module's directory as an entry: cleaned, relative, `.` for the directory.
|
||||||
|
func cleanEntry(p string) string {
|
||||||
|
c := strings.Trim(path.Clean("/"+filepath.ToSlash(p)), "/")
|
||||||
|
if c == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// orNoTree is why a build has no source fingerprint, for its log.
|
// orNoTree is why a build has no source fingerprint, for its log.
|
||||||
func orNoTree(why string) string {
|
func orNoTree(why string) string {
|
||||||
if why == "" {
|
if why == "" {
|
||||||
@@ -648,15 +706,51 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
} else if src != nil {
|
} else if src != nil {
|
||||||
src.contexts[a.Name] = t
|
src.contexts[a.Name] = t
|
||||||
}
|
}
|
||||||
// docker build accepts -f outside the context it is given; the recipe stays exactly
|
buildDir = cloned
|
||||||
// where it was read from and validated against, absolute so the working directory
|
if t, _ := trunkOf(ctx, run, cloned, "HEAD"); t == "" || !atTrunkHead(ctx, run, cloned, "HEAD", t) {
|
||||||
// switching to the cloned context does not change which file that is.
|
src.contextOffHead()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// docker build accepts -f outside the context it is given; the recipe stays exactly where it was
|
||||||
|
// read from and validated against, absolute so a context elsewhere does not change which file
|
||||||
|
// that is.
|
||||||
|
if buildDir != tree || a.Compiles != "" {
|
||||||
absRecipe, err := filepath.Abs(filepath.Join(tree, a.From))
|
absRecipe, err := filepath.Abs(filepath.Join(tree, a.From))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err)
|
return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err)
|
||||||
}
|
}
|
||||||
recipePath = absRecipe
|
recipePath = absRecipe
|
||||||
buildDir = cloned
|
}
|
||||||
|
// **An image that compiles a Go program is handed its build source and nothing else** (novox/hq
|
||||||
|
// ADR 0267 rules 1 and 3): the program's import closure, read from the context it is built in, so a
|
||||||
|
// merge elsewhere in that repository is no change to it — and a recipe that copies a file outside
|
||||||
|
// it fails here, naming the file, rather than building from something no merge is mapped onto.
|
||||||
|
if a.Compiles != "" {
|
||||||
|
paths, err := GoBuildSource(buildDir, a.Compiles)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Built{}, fmt.Errorf("%s: %s compiles %s, whose build source cannot be read: %w",
|
||||||
|
module, a.Name, a.Compiles, err)
|
||||||
|
}
|
||||||
|
narrowed := filepath.Join(workspace, "narrow-"+a.Name)
|
||||||
|
sum, err := narrowTree(buildDir, narrowed, paths)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Built{}, fmt.Errorf("%s: handing %s its build source: %w", module, a.Name, err)
|
||||||
|
}
|
||||||
|
say("image", "%s is handed its build source: %d path(s) of %s", a.Name, len(paths), a.Compiles)
|
||||||
|
if a.Context != nil {
|
||||||
|
src.contexts[a.Name] = sum
|
||||||
|
src.readIn(*a.Context, paths)
|
||||||
|
} else {
|
||||||
|
src.ownHas(paths...)
|
||||||
|
}
|
||||||
|
buildDir = narrowed
|
||||||
|
} else if a.Context != nil {
|
||||||
|
src.readWhole(*a.Context)
|
||||||
|
}
|
||||||
|
if a.Compiles != "" || a.Context != nil {
|
||||||
|
src.ownHas(cleanEntry(a.From))
|
||||||
|
} else {
|
||||||
|
src.ownWhole()
|
||||||
}
|
}
|
||||||
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
||||||
if a.Target != "" {
|
if a.Target != "" {
|
||||||
@@ -708,6 +802,18 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
if src != nil {
|
if src != nil {
|
||||||
src.toolchains[a.Name] = toolchainOf(chain, base)
|
src.toolchains[a.Name] = toolchainOf(chain, base)
|
||||||
}
|
}
|
||||||
|
// A Go program's build source is its import closure (novox/hq ADR 0267 rule 1). Not read, it is the
|
||||||
|
// module's whole directory, as before — said, so the wider plan has a reason a person can find.
|
||||||
|
if chain.Language == "go" {
|
||||||
|
if paths, err := GoBuildSource(tree, a.From); err != nil {
|
||||||
|
say("bundle", "%s's build source is its whole directory: %v", a.Name, err)
|
||||||
|
src.ownWhole()
|
||||||
|
} else {
|
||||||
|
src.ownHas(paths...)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
src.ownWhole()
|
||||||
|
}
|
||||||
if chain.Language == "typescript" {
|
if chain.Language == "typescript" {
|
||||||
if own, _ := ownDependencies(tree); len(own) > 0 {
|
if own, _ := ownDependencies(tree); len(own) > 0 {
|
||||||
src.notPinned(a.Name + " resolves packages of its own at build time")
|
src.notPinned(a.Name + " resolves packages of its own at build time")
|
||||||
@@ -754,6 +860,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
// there is no Publisher call — the container itself publishes, with the credential the
|
// there is no Publisher call — the container itself publishes, with the credential the
|
||||||
// build was handed.
|
// build was handed.
|
||||||
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
||||||
|
src.ownWhole()
|
||||||
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
||||||
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -763,6 +870,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||||
|
|
||||||
case catalogue.ArtifactArchive:
|
case catalogue.ArtifactArchive:
|
||||||
|
src.ownHas(cleanEntry(a.From) + "/**")
|
||||||
body, err := pack(filepath.Join(tree, a.From))
|
body, err := pack(filepath.Join(tree, a.From))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
||||||
|
|||||||
@@ -0,0 +1,528 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Go program's build source (novox/hq ADR 0267 rule 1): the files it is built from, derived from its
|
||||||
|
// import closure rather than listed by hand, because a list drifts from the imports it describes and a
|
||||||
|
// path missing from it is a real change missed — worse than a needless rebuild.
|
||||||
|
//
|
||||||
|
// **The closure read here is never narrower than `go list -deps`.** Every .go file of a package that is
|
||||||
|
// not a test is read, whatever its build constraint, so the imports are the union over every system and
|
||||||
|
// tag; a directory is held whole (but for its tests), so a file added to a package is in it; an embed is
|
||||||
|
// held by the directory its pattern starts in, everything below it. Read with the standard library's
|
||||||
|
// parser and no toolchain: the build machine carries none, and a closure that needed the network to
|
||||||
|
// read would be one a build could not say offline.
|
||||||
|
//
|
||||||
|
// A build source is a list of entries, relative to the root the build sees:
|
||||||
|
//
|
||||||
|
// dir/ a Go package's directory: every file directly in it but its tests (`*_test.go`)
|
||||||
|
// dir/** everything below a directory (an embed)
|
||||||
|
// ** the whole tree
|
||||||
|
// file one file
|
||||||
|
//
|
||||||
|
// The root package's directory is `./`.
|
||||||
|
|
||||||
|
// GoPackageDirEntry is the entry for a Go package's directory.
|
||||||
|
func goPackageDirEntry(dir string) string {
|
||||||
|
if dir == "" || dir == "." {
|
||||||
|
return "./"
|
||||||
|
}
|
||||||
|
return dir + "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceHolds is whether a changed file — a path relative to the root the build source was read in — is
|
||||||
|
// in that build source.
|
||||||
|
func SourceHolds(entries []string, file string) bool {
|
||||||
|
file = strings.TrimPrefix(path.Clean("/"+strings.TrimSpace(file)), "/")
|
||||||
|
for _, e := range entries {
|
||||||
|
switch {
|
||||||
|
case e == "**":
|
||||||
|
return true
|
||||||
|
case strings.HasSuffix(e, "/**"):
|
||||||
|
dir := strings.TrimSuffix(e, "/**")
|
||||||
|
if dir == "." || dir == "" || file == dir || strings.HasPrefix(file, dir+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
case strings.HasSuffix(e, "/"):
|
||||||
|
dir := strings.TrimSuffix(e, "/")
|
||||||
|
parent := path.Dir(file)
|
||||||
|
if (dir == "." && parent == ".") || parent == dir {
|
||||||
|
if !strings.HasSuffix(file, "_test.go") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case e == file:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoBuildSource is the build source of the Go program whose main package is pkg, a directory relative to
|
||||||
|
// root: the directories of every package of its import closure inside root, the embeds those packages
|
||||||
|
// name, its module's go.mod, go.sum and vendor/modules.txt, and a go.work wherever one would be read. An
|
||||||
|
// entry for a file that does not exist is kept: creating it is a change to the build.
|
||||||
|
//
|
||||||
|
// Refused — so the build source is not narrowed, and nothing is missed — when the closure cannot be told
|
||||||
|
// from the files: no go.mod holds the package, a go.work is present, a local replace leaves root, a file
|
||||||
|
// does not parse, or a cgo preamble reaches outside its directory.
|
||||||
|
func GoBuildSource(root, pkg string) ([]string, error) {
|
||||||
|
root, err := filepath.Abs(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rel := path.Clean(strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(pkg)), "/"))
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, "../") {
|
||||||
|
return nil, fmt.Errorf("the package %q leaves the tree it is built from", pkg)
|
||||||
|
}
|
||||||
|
if info, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil || !info.IsDir() {
|
||||||
|
return nil, fmt.Errorf("%q is not a directory of the tree it is built from", pkg)
|
||||||
|
}
|
||||||
|
// The module holding the package: the nearest go.mod at or above it, within root.
|
||||||
|
modRoot := ""
|
||||||
|
for dir := rel; ; dir = path.Dir(dir) {
|
||||||
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.mod")); err == nil {
|
||||||
|
modRoot = dir
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if dir == "." {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if modRoot == "" {
|
||||||
|
return nil, fmt.Errorf("no go.mod holds %q within the tree it is built from", pkg)
|
||||||
|
}
|
||||||
|
entries := map[string]bool{}
|
||||||
|
file := func(dir, name string) {
|
||||||
|
entries[strings.TrimPrefix(path.Join(dir, name), "./")] = true
|
||||||
|
}
|
||||||
|
file(modRoot, "go.mod")
|
||||||
|
file(modRoot, "go.sum")
|
||||||
|
file(modRoot, "vendor/modules.txt")
|
||||||
|
// A workspace changes how every import resolves; one present is not read past, one created later is a
|
||||||
|
// change to the build.
|
||||||
|
for dir := modRoot; ; dir = path.Dir(dir) {
|
||||||
|
file(dir, "go.work")
|
||||||
|
file(dir, "go.work.sum")
|
||||||
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.work")); err == nil {
|
||||||
|
return nil, fmt.Errorf("%s holds a go.work, and a workspace's imports are not read here", path.Join(dir, "go.work"))
|
||||||
|
}
|
||||||
|
if dir == "." {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
modPath, replaces, err := readGoMod(filepath.Join(root, filepath.FromSlash(modRoot), "go.mod"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
vendored := false
|
||||||
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(modRoot), "vendor", "modules.txt")); err == nil {
|
||||||
|
vendored = true
|
||||||
|
}
|
||||||
|
// resolve is the directories, relative to root, an import may be read from: none for the standard
|
||||||
|
// library and for a module outside the tree, which go.mod and go.sum pin. A vendored module replaced
|
||||||
|
// by a local directory is both — vendor/ under -mod=vendor, the directory under -mod=mod — and both
|
||||||
|
// are held, so neither way of building it is missed.
|
||||||
|
resolve := func(importPath string) ([]string, error) {
|
||||||
|
within := func(prefix, dir string) (string, bool) {
|
||||||
|
if importPath == prefix {
|
||||||
|
return dir, true
|
||||||
|
}
|
||||||
|
if rest, ok := strings.CutPrefix(importPath, prefix+"/"); ok {
|
||||||
|
return path.Join(dir, rest), true
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if dir, ok := within(modPath, modRoot); ok {
|
||||||
|
return []string{dir}, nil
|
||||||
|
}
|
||||||
|
var dirs []string
|
||||||
|
for _, r := range replaces {
|
||||||
|
if sub, ok := within(r.from, ""); ok {
|
||||||
|
target := path.Clean(path.Join(modRoot, r.to))
|
||||||
|
if target == ".." || strings.HasPrefix(target, "../") {
|
||||||
|
return nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", r.from, r.to)
|
||||||
|
}
|
||||||
|
dirs = append(dirs, path.Join(target, sub))
|
||||||
|
// Its go.mod states what it requires, read when it is built from there.
|
||||||
|
entries[path.Join(target, "go.mod")] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
first, _, _ := strings.Cut(importPath, "/")
|
||||||
|
if len(dirs) == 0 && !strings.Contains(first, ".") {
|
||||||
|
return nil, nil // the standard library
|
||||||
|
}
|
||||||
|
if vendored {
|
||||||
|
dirs = append(dirs, path.Join(modRoot, "vendor", importPath))
|
||||||
|
}
|
||||||
|
return dirs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[string]bool{}
|
||||||
|
queue := []string{rel}
|
||||||
|
for len(queue) > 0 {
|
||||||
|
dir := queue[0]
|
||||||
|
queue = queue[1:]
|
||||||
|
if seen[dir] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[dir] = true
|
||||||
|
entries[goPackageDirEntry(dir)] = true
|
||||||
|
// A go.mod made between the module's root and a package moves that package out of the module.
|
||||||
|
for up := dir; up != modRoot && up != "." && up != "/" && !strings.HasPrefix(up, "../"); up = path.Dir(up) {
|
||||||
|
file(up, "go.mod")
|
||||||
|
}
|
||||||
|
listing, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir)))
|
||||||
|
if err != nil {
|
||||||
|
// A package that is not there fails the build that imports it; its directory is held, so
|
||||||
|
// adding it is a change.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, f := range listing {
|
||||||
|
name := f.Name()
|
||||||
|
// **C and assembly beside Go** may include files from below the package's directory: the
|
||||||
|
// directory is held whole, and an include reaching above it is refused.
|
||||||
|
if !f.IsDir() && nativeSource(name) {
|
||||||
|
entries[strings.TrimPrefix(dir+"/**", "./")] = true
|
||||||
|
if dir == "." {
|
||||||
|
entries["**"] = true
|
||||||
|
}
|
||||||
|
if err := includesStayWithin(filepath.Join(root, filepath.FromSlash(dir), name)); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
full := filepath.Join(root, filepath.FromSlash(dir), name)
|
||||||
|
imports, embeds, err := goFileReads(full)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
||||||
|
}
|
||||||
|
for _, ip := range imports {
|
||||||
|
targets, err := resolve(ip)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, target := range targets {
|
||||||
|
if !seen[target] {
|
||||||
|
queue = append(queue, target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, e := range embeds {
|
||||||
|
entries[path.Join(dir, e)+"/**"] = true
|
||||||
|
if !strings.ContainsAny(e, "*?[\\") {
|
||||||
|
entries[path.Join(dir, e)] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(entries))
|
||||||
|
for e := range entries {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// goReplace is one local replacement in go.mod: an import path read from a directory.
|
||||||
|
type goReplace struct{ from, to string }
|
||||||
|
|
||||||
|
// readGoMod is a go.mod's module path and its replacements by a local directory. A replacement by another
|
||||||
|
// module version is resolved by go.sum, which the build source holds.
|
||||||
|
func readGoMod(file string) (string, []goReplace, error) {
|
||||||
|
f, err := os.Open(file)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
var module string
|
||||||
|
var replaces []goReplace
|
||||||
|
inReplace := false
|
||||||
|
scanner := bufio.NewScanner(f)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if i := strings.Index(line, "//"); i >= 0 {
|
||||||
|
line = line[:i]
|
||||||
|
}
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
switch {
|
||||||
|
case line == "":
|
||||||
|
continue
|
||||||
|
case inReplace && line == ")":
|
||||||
|
inReplace = false
|
||||||
|
continue
|
||||||
|
case strings.HasPrefix(line, "module "):
|
||||||
|
module = unquoteGoMod(strings.TrimSpace(strings.TrimPrefix(line, "module")))
|
||||||
|
continue
|
||||||
|
case line == "replace (":
|
||||||
|
inReplace = true
|
||||||
|
continue
|
||||||
|
case strings.HasPrefix(line, "replace "):
|
||||||
|
line = strings.TrimSpace(strings.TrimPrefix(line, "replace"))
|
||||||
|
case !inReplace:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
left, right, found := strings.Cut(line, "=>")
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from := strings.Fields(left)
|
||||||
|
to := strings.Fields(right)
|
||||||
|
if len(from) == 0 || len(to) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target := unquoteGoMod(to[0])
|
||||||
|
// A local replacement is a path: ./, ../ or absolute. Anything else names a module version.
|
||||||
|
if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || target == "." || target == ".." {
|
||||||
|
replaces = append(replaces, goReplace{from: unquoteGoMod(from[0]), to: target})
|
||||||
|
} else if strings.HasPrefix(target, "/") {
|
||||||
|
return "", nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", from[0], target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return "", nil, fmt.Errorf("%s names no module", file)
|
||||||
|
}
|
||||||
|
// The longest replacement first, so a replaced sub-path wins over its parent.
|
||||||
|
sort.SliceStable(replaces, func(i, j int) bool { return len(replaces[i].from) > len(replaces[j].from) })
|
||||||
|
return module, replaces, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func unquoteGoMod(s string) string {
|
||||||
|
if u, err := strconv.Unquote(s); err == nil {
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// goFileReads is what one Go file makes its build read: the packages it imports, and the directories its
|
||||||
|
// //go:embed patterns start in, relative to its own directory ("." for the directory itself).
|
||||||
|
//
|
||||||
|
// **A cgo preamble reaching outside its directory is refused**: a header included by a relative path, or
|
||||||
|
// a flag naming ${SRCDIR}/.., is a file of the build no import names. Inside the directory it is held
|
||||||
|
// already.
|
||||||
|
func goFileReads(file string) (imports, embeds []string, err error) {
|
||||||
|
src, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
parsed, err := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.ParseComments)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
for _, spec := range parsed.Imports {
|
||||||
|
ip, err := strconv.Unquote(spec.Path.Value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if ip == "C" {
|
||||||
|
// The preamble is the comment before the import; only its #include and #cgo lines read files.
|
||||||
|
for _, cg := range parsed.Comments {
|
||||||
|
if cg.End() > spec.Pos() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(cg.Text(), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if (strings.HasPrefix(line, "#include") || strings.HasPrefix(line, "#cgo")) && strings.Contains(line, "..") {
|
||||||
|
return nil, nil, errors.New("its cgo preamble names a path outside its directory: " + line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A cgo file may include from below its directory: the directory is held whole.
|
||||||
|
embeds = append(embeds, ".")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
imports = append(imports, ip)
|
||||||
|
}
|
||||||
|
// //go:embed directives may stand anywhere in the file, so the whole text is read for them.
|
||||||
|
scanner := bufio.NewScanner(strings.NewReader(string(src)))
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
rest, ok := strings.CutPrefix(line, "//go:embed")
|
||||||
|
if !ok || (rest != "" && rest[0] != ' ' && rest[0] != '\t') {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
patterns, err := embedPatterns(rest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
for _, p := range patterns {
|
||||||
|
embeds = append(embeds, embedRoot(p))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return imports, embeds, scanner.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// nativeSource is a file the Go command compiles or links beside Go: C, C++, Objective-C, Fortran,
|
||||||
|
// assembly, their headers and a system object.
|
||||||
|
func nativeSource(name string) bool {
|
||||||
|
switch strings.ToLower(path.Ext(name)) {
|
||||||
|
case ".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".m", ".s", ".sx", ".f", ".f90", ".for", ".syso":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// includesStayWithin refuses a native source whose #include names a path above its directory.
|
||||||
|
func includesStayWithin(file string) error {
|
||||||
|
body, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(string(body), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(line, "#") && strings.Contains(line, "include") && strings.Contains(line, "..") {
|
||||||
|
return errors.New("it includes a path outside its directory: " + line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// embedPatterns splits a //go:embed line's patterns: separated by spaces, each possibly quoted.
|
||||||
|
func embedPatterns(s string) ([]string, error) {
|
||||||
|
var out []string
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
for s != "" {
|
||||||
|
var p string
|
||||||
|
switch s[0] {
|
||||||
|
case '"', '`':
|
||||||
|
q, err := strconv.QuotedPrefix(s)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("an embed pattern does not parse: %w", err)
|
||||||
|
}
|
||||||
|
if p, err = strconv.Unquote(q); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s = s[len(q):]
|
||||||
|
default:
|
||||||
|
end := strings.IndexAny(s, " \t")
|
||||||
|
if end < 0 {
|
||||||
|
end = len(s)
|
||||||
|
}
|
||||||
|
p, s = s[:end], s[end:]
|
||||||
|
}
|
||||||
|
out = append(out, p)
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// embedRoot is the directory an embed pattern starts in, relative to the package: its leading elements
|
||||||
|
// without a wildcard. A pattern naming a file or a directory outright is held as itself.
|
||||||
|
func embedRoot(pattern string) string {
|
||||||
|
pattern = strings.TrimPrefix(pattern, "all:")
|
||||||
|
var kept []string
|
||||||
|
for _, el := range strings.Split(pattern, "/") {
|
||||||
|
if strings.ContainsAny(el, "*?[\\") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
kept = append(kept, el)
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return path.Clean(strings.Join(kept, "/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// narrowTree copies into dst the files of src a build source holds, and nothing else — never `.git` — and
|
||||||
|
// returns a fingerprint of what it copied: each file's path, mode and content, hashed in path order. **A
|
||||||
|
// build handed only its build source cannot read past it** (novox/hq ADR 0267 rule 3): a recipe that
|
||||||
|
// copies a file outside it fails, naming the file, where it would have built and been missed.
|
||||||
|
func narrowTree(src, dst string, entries []string) (string, error) {
|
||||||
|
if err := os.RemoveAll(dst); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(src, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
if d.IsDir() {
|
||||||
|
if d.Name() == ".git" {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !SourceHolds(entries, rel) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := filepath.Join(dst, filepath.FromSlash(rel))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := d.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&fs.ModeSymlink != 0 {
|
||||||
|
// A link in the repository is copied as the link it is, and what it names said in the
|
||||||
|
// fingerprint.
|
||||||
|
target, err := os.Readlink(p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lines = append(lines, fmt.Sprintf("%s link %s", rel, target))
|
||||||
|
return os.Symlink(target, out)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
in, err := os.Open(p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
w, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sum := sha256.New()
|
||||||
|
if _, err := io.Copy(io.MultiWriter(w, sum), in); err != nil {
|
||||||
|
w.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lines = append(lines, fmt.Sprintf("%s %o %s", rel, info.Mode().Perm()&0o111, hex.EncodeToString(sum.Sum(nil))))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sort.Strings(lines)
|
||||||
|
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||||
|
return "narrow:" + hex.EncodeToString(sum[:]), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,318 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Go program's build source is its import closure (novox/hq ADR 0267 rule 1): never narrower than what
|
||||||
|
// `go build` reads, whatever the system, the tags, the vendoring or the embeds.
|
||||||
|
|
||||||
|
// aGoTree writes files under a fresh directory and returns it.
|
||||||
|
func aGoTree(t *testing.T, files map[string]string) string {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, body := range files {
|
||||||
|
full := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
// aProgram is a module whose program imports its own packages, a vendored module, a local replacement
|
||||||
|
// that is vendored too, an embed and a package only one system builds; beside them, a package nothing
|
||||||
|
// imports and one only a test imports.
|
||||||
|
var aProgram = map[string]string{
|
||||||
|
"go.mod": "module example.com/fix\n\ngo 1.22\n\nrequire (\n\texample.org/dep v1.0.0\n\texample.net/local v0.0.0\n)\n\n" +
|
||||||
|
"replace example.net/local => ./third_party/local\n",
|
||||||
|
"go.sum": "",
|
||||||
|
"vendor/modules.txt": "# example.net/local v0.0.0 => ./third_party/local\n## explicit; go 1.22\nexample.net/local/pkg\n" +
|
||||||
|
"# example.org/dep v1.0.0\n## explicit; go 1.22\nexample.org/dep\nexample.org/dep/sub\n# example.net/local => ./third_party/local\n",
|
||||||
|
"vendor/example.org/dep/dep.go": "package dep\n\nimport _ \"example.org/dep/sub\"\n",
|
||||||
|
"vendor/example.org/dep/sub/sub.go": "package sub\n",
|
||||||
|
"vendor/example.org/other/other.go": "package other\n",
|
||||||
|
"vendor/example.net/local/pkg/p.go": "package pkg\n",
|
||||||
|
"third_party/local/go.mod": "module example.net/local\n\ngo 1.22\n",
|
||||||
|
"third_party/local/pkg/p.go": "package pkg\n",
|
||||||
|
"cmd/prog/main.go": "package main\n\nimport (\n\t\"fmt\"\n\n\t_ \"example.com/fix/emb\"\n\t\"example.com/fix/lib\"\n" +
|
||||||
|
"\t_ \"example.net/local/pkg\"\n\t_ \"example.org/dep\"\n)\n\nfunc main() { fmt.Println(lib.X) }\n",
|
||||||
|
"lib/lib.go": "package lib\n\nconst X = 1\n",
|
||||||
|
"lib/lib_plan9.go": "//go:build plan9\n\npackage lib\n\nimport _ \"example.com/fix/plan9only\"\n",
|
||||||
|
"lib/lib_test.go": "package lib\n\nimport _ \"example.com/fix/testonly\"\n",
|
||||||
|
"emb/emb_amd64.s": "",
|
||||||
|
"lib/sub/sub.go": "package sub\n",
|
||||||
|
"plan9only/p.go": "package plan9only\n",
|
||||||
|
"testonly/t.go": "package testonly\n",
|
||||||
|
"unrelated/u.go": "package unrelated\n",
|
||||||
|
"emb/emb.go": "package emb\n\nimport \"embed\"\n\n//go:embed static/*\nvar Static embed.FS\n\n" +
|
||||||
|
"//go:embed \"a b.txt\"\nvar Text string\n",
|
||||||
|
"emb/static/index.html": "x",
|
||||||
|
"emb/static/deep/style.css": "x",
|
||||||
|
"emb/a b.txt": "x",
|
||||||
|
"README.md": "x",
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) {
|
||||||
|
root := aGoTree(t, aProgram)
|
||||||
|
got, err := GoBuildSource(root, "cmd/prog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
file string
|
||||||
|
held bool
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{"cmd/prog/main.go", true, "the program itself"},
|
||||||
|
{"cmd/prog/helper.go", true, "a file added to the program's package"},
|
||||||
|
{"lib/lib.go", true, "a package it imports"},
|
||||||
|
{"emb/emb_amd64.s", true, "assembly beside a package's Go"},
|
||||||
|
{"lib/lib_plan9.go", true, "a file one system builds"},
|
||||||
|
{"plan9only/p.go", true, "what a file one system builds imports"},
|
||||||
|
{"emb/static/index.html", true, "an embedded file"},
|
||||||
|
{"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"},
|
||||||
|
{"emb/a b.txt", true, "an embed named outright, quoted"},
|
||||||
|
{"vendor/example.org/dep/dep.go", true, "a vendored package it imports"},
|
||||||
|
{"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"},
|
||||||
|
{"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"},
|
||||||
|
{"third_party/local/pkg/p.go", true, "a replaced module, at its directory"},
|
||||||
|
{"third_party/local/go.mod", true, "a replaced module's requirements"},
|
||||||
|
{"go.mod", true, "the module's requirements"},
|
||||||
|
{"go.sum", true, "the module's sums"},
|
||||||
|
{"vendor/modules.txt", true, "the vendored modules"},
|
||||||
|
{"go.work", true, "a workspace, were one made"},
|
||||||
|
{"lib/lib_test.go", false, "a test is not built"},
|
||||||
|
{"testonly/t.go", false, "a package only a test imports"},
|
||||||
|
{"lib/sub/sub.go", false, "a package below an imported one, not imported"},
|
||||||
|
{"unrelated/u.go", false, "a package nothing imports"},
|
||||||
|
{"vendor/example.org/other/other.go", false, "a vendored package nothing imports"},
|
||||||
|
{"README.md", false, "a file no build reads"},
|
||||||
|
} {
|
||||||
|
if SourceHolds(got, c.file) != c.held {
|
||||||
|
t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds,
|
||||||
|
// is held. Where no go command is at hand, said, not passed.
|
||||||
|
goCmd, err := exec.LookPath("go")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("NOT COMPARED: no go command to list the closure with")
|
||||||
|
}
|
||||||
|
list := exec.Command(goCmd, "list", "-deps", "-f",
|
||||||
|
`{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}}
|
||||||
|
{{end}}{{range .SFiles}}{{$d}}/{{.}}
|
||||||
|
{{end}}{{range .EmbedFiles}}{{$d}}/{{.}}
|
||||||
|
{{end}}{{end}}`, "./cmd/prog")
|
||||||
|
list.Dir = root
|
||||||
|
list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64")
|
||||||
|
out, err := list.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("go list: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
real, _ := filepath.EvalSymlinks(root)
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(real, line)
|
||||||
|
if err != nil || strings.HasPrefix(rel, "..") {
|
||||||
|
rel, _ = filepath.Rel(root, line)
|
||||||
|
}
|
||||||
|
if !SourceHolds(got, filepath.ToSlash(rel)) {
|
||||||
|
t.Errorf("go list builds %s, and the build source does not hold it", rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file added to the program's import closure moves its build source with it: the closure read again
|
||||||
|
// grows by the package.
|
||||||
|
func TestTheBuildSourceGrowsWithAnImport(t *testing.T) {
|
||||||
|
files := map[string]string{}
|
||||||
|
for k, v := range aProgram {
|
||||||
|
files[k] = v
|
||||||
|
}
|
||||||
|
before, err := GoBuildSource(aGoTree(t, files), "cmd/prog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if SourceHolds(before, "unrelated/u.go") {
|
||||||
|
t.Fatal("held before it was imported")
|
||||||
|
}
|
||||||
|
files["cmd/prog/more.go"] = "package main\n\nimport _ \"example.com/fix/unrelated\"\n"
|
||||||
|
after, err := GoBuildSource(aGoTree(t, files), "cmd/prog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !SourceHolds(after, "unrelated/u.go") {
|
||||||
|
t.Fatalf("an import added did not grow the build source: %v", after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What cannot be read is refused, so the build source is not narrowed and nothing is missed.
|
||||||
|
func TestABuildSourceThatCannotBeReadIsRefused(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
files map[string]string
|
||||||
|
pkg string
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"no go.mod", map[string]string{"cmd/p/main.go": "package main\n"}, "cmd/p", "no go.mod"},
|
||||||
|
{"a workspace", map[string]string{"go.mod": "module x\n", "go.work": "go 1.22\n", "p/main.go": "package main\n"},
|
||||||
|
"p", "go.work"},
|
||||||
|
{"a local replacement outside the tree", map[string]string{
|
||||||
|
"go.mod": "module x\n\nreplace y => ../y\n", "p/main.go": "package main\n\nimport _ \"y\"\n"}, "p", "outside"},
|
||||||
|
{"a cgo header outside the directory", map[string]string{
|
||||||
|
"go.mod": "module x\n", "p/main.go": "package main\n\n// #include \"../h/h.h\"\nimport \"C\"\n"}, "p", "cgo"},
|
||||||
|
{"an assembly include above its directory", map[string]string{"go.mod": "module x\n", "p/main.go": "package main\n",
|
||||||
|
"p/a_amd64.s": "#include \"../h/textflag.h\"\n"}, "p", "outside"},
|
||||||
|
{"a file that does not parse", map[string]string{"go.mod": "module x\n", "p/main.go": "package main\n\nimport (\n"},
|
||||||
|
"p", "main.go"},
|
||||||
|
{"a package that leaves the tree", map[string]string{"go.mod": "module x\n"}, "../elsewhere", "leaves"},
|
||||||
|
} {
|
||||||
|
_, err := GoBuildSource(aGoTree(t, c.files), c.pkg)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("%s: %v, wanted a refusal saying %q", c.what, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A cgo header in the package's own directory is held already, and is no refusal.
|
||||||
|
if _, err := GoBuildSource(aGoTree(t, map[string]string{"go.mod": "module x\n",
|
||||||
|
"p/main.go": "package main\n\n// #include \"h.h\"\nimport \"C\"\n"}), "p"); err != nil {
|
||||||
|
t.Errorf("a header in the package's directory was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABuildSourceHoldsWhatItsEntriesSay(t *testing.T) {
|
||||||
|
entries := []string{"./", "cmd/p/", "web/**", "go.mod"}
|
||||||
|
for file, want := range map[string]bool{
|
||||||
|
"main.go": true, "main_test.go": false, "cmd/p/x.go": true, "cmd/p/x_test.go": false, "cmd/p/sub/y.go": false,
|
||||||
|
"cmd/px/x.go": false, "web/a/b/c.css": true, "web": true, "webx/a": false, "go.mod": true, "docs/x.md": false,
|
||||||
|
"/cmd/p/x.go": true, "cmd/p/../q/x.go": false,
|
||||||
|
} {
|
||||||
|
if SourceHolds(entries, file) != want {
|
||||||
|
t.Errorf("%s: held %v, wanted %v", file, !want, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !SourceHolds([]string{"**"}, "anything/at/all") {
|
||||||
|
t.Error("the whole tree does not hold a file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A build handed its build source reads nothing else** (rule 3): the narrowed tree holds the source's
|
||||||
|
// files and no others — never .git — and its fingerprint changes with them and only with them.
|
||||||
|
func TestANarrowedTreeHoldsTheBuildSourceAndNothingElse(t *testing.T) {
|
||||||
|
files := map[string]string{}
|
||||||
|
for k, v := range aProgram {
|
||||||
|
files[k] = v
|
||||||
|
}
|
||||||
|
files[".git/HEAD"] = "ref: refs/heads/main\n"
|
||||||
|
src := aGoTree(t, files)
|
||||||
|
entries, err := GoBuildSource(src, "cmd/prog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dst := filepath.Join(t.TempDir(), "narrow")
|
||||||
|
one, err := narrowTree(src, dst, entries)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for file, want := range map[string]bool{"cmd/prog/main.go": true, "lib/lib.go": true, "emb/static/deep/style.css": true,
|
||||||
|
"lib/lib_test.go": false, "unrelated/u.go": false, "README.md": false, ".git/HEAD": false} {
|
||||||
|
_, err := os.Stat(filepath.Join(dst, filepath.FromSlash(file)))
|
||||||
|
if (err == nil) != want {
|
||||||
|
t.Errorf("%s: copied %v, wanted %v", file, err == nil, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Outside the build source: one fingerprint.
|
||||||
|
files["README.md"] = "changed"
|
||||||
|
files["unrelated/u.go"] = "package unrelated\n\nconst Changed = 1\n"
|
||||||
|
again, err := narrowTree(aGoTree(t, files), filepath.Join(t.TempDir(), "n"), entries)
|
||||||
|
if err != nil || again != one {
|
||||||
|
t.Fatalf("a change outside the build source changed its fingerprint: %s %s %v", one, again, err)
|
||||||
|
}
|
||||||
|
// Inside it: another.
|
||||||
|
files["lib/lib.go"] = "package lib\n\nconst X = 2\n"
|
||||||
|
moved, err := narrowTree(aGoTree(t, files), filepath.Join(t.TempDir(), "n"), entries)
|
||||||
|
if err != nil || moved == one {
|
||||||
|
t.Fatalf("a change inside the build source kept its fingerprint: %s %v", moved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// This repository's own programs: the route proxy's build source is not the controller's, and neither
|
||||||
|
// holds the other's command.
|
||||||
|
func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) {
|
||||||
|
root := filepath.Join("..", "..")
|
||||||
|
proxy, err := GoBuildSource(root, "examples/route-proxy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
controller, err := GoBuildSource(root, "cmd/mesh-controller")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
builder, err := GoBuildSource(root, "cmd/mesh-builder")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
entries []string
|
||||||
|
name string
|
||||||
|
file string
|
||||||
|
held bool
|
||||||
|
}{
|
||||||
|
{proxy, "the route proxy", "examples/route-proxy/main.go", true},
|
||||||
|
{proxy, "the route proxy", "internal/broker/broker.go", true},
|
||||||
|
{proxy, "the route proxy", "cmd/mesh-controller/main.go", false},
|
||||||
|
{proxy, "the route proxy", "internal/conditions/condition.go", false},
|
||||||
|
{proxy, "the route proxy", "README.md", false},
|
||||||
|
{controller, "the controller", "cmd/mesh-controller/main.go", true},
|
||||||
|
{controller, "the controller", "internal/conditions/condition.go", true},
|
||||||
|
{controller, "the controller", "internal/inventory/migrations/0001-nodes.sql", true},
|
||||||
|
{controller, "the controller", "examples/route-proxy/main.go", false},
|
||||||
|
{controller, "the controller", "cmd/mesh-builder/main.go", false},
|
||||||
|
{controller, "the controller", "README.md", false},
|
||||||
|
{builder, "the build seat's program", "cmd/mesh-builder/main.go", true},
|
||||||
|
{builder, "the build seat's program", "internal/conditions/condition.go", false},
|
||||||
|
{builder, "the build seat's program", "cmd/mesh-controller/main.go", false},
|
||||||
|
} {
|
||||||
|
if SourceHolds(c.entries, c.file) != c.held {
|
||||||
|
t.Errorf("%s: %s held %v, wanted %v", c.name, c.file, !c.held, c.held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// C or assembly beside Go holds its package's directory whole — an include may name a file below it — and a
|
||||||
|
// go.mod made between the module's root and a package is a change.
|
||||||
|
func TestNativeSourcesHoldTheirDirectoryWhole(t *testing.T) {
|
||||||
|
got, err := GoBuildSource(aGoTree(t, map[string]string{"go.mod": "module x\n", "cmd/p/main.go": "package main\n\nimport _ \"x/lib/asm\"\n",
|
||||||
|
"lib/asm/a.go": "package asm\n", "lib/asm/a_amd64.s": "#include \"inc/textflag.h\"\n", "lib/asm/inc/textflag.h": ""}), "cmd/p")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for file, want := range map[string]bool{"lib/asm/inc/textflag.h": true, "lib/asm/a_amd64.s": true, "lib/go.mod": true,
|
||||||
|
"lib/asm/go.mod": true, "cmd/go.mod": true, "other/go.mod": false} {
|
||||||
|
if SourceHolds(got, file) != want {
|
||||||
|
t.Errorf("%s: held %v, wanted %v (%v)", file, !want, want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cgo file may include from below its directory with nothing native beside it: its directory is held whole.
|
||||||
|
func TestACgoFileHoldsItsDirectoryWhole(t *testing.T) {
|
||||||
|
got, err := GoBuildSource(aGoTree(t, map[string]string{"go.mod": "module x\n",
|
||||||
|
"p/main.go": "package main\n\n// #include \"inc/x.h\"\nimport \"C\"\n", "p/inc/x.h": ""}), "p")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !SourceHolds(got, "p/inc/x.h") {
|
||||||
|
t.Fatalf("a header a cgo preamble includes is not held: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+145
-1
@@ -9,6 +9,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A build's source fingerprint: what it was made from, hashed (novox/hq issue 280).
|
// A build's source fingerprint: what it was made from, hashed (novox/hq issue 280).
|
||||||
@@ -49,10 +51,152 @@ type sourceInputs struct {
|
|||||||
toolchains map[string]string
|
toolchains map[string]string
|
||||||
// unpinned is why this build has no fingerprint: empty when it has one.
|
// unpinned is why this build has no fingerprint: empty when it has one.
|
||||||
unpinned string
|
unpinned string
|
||||||
|
|
||||||
|
// prefix is the module's directory within its repository, empty at the root.
|
||||||
|
prefix string
|
||||||
|
// own is the module's build source in its own repository, relative to the module's directory, and
|
||||||
|
// whole when an artifact's is not known (novox/hq ADR 0267).
|
||||||
|
own map[string]bool
|
||||||
|
ownIsAll bool
|
||||||
|
// read is, per context (repository and ref), the build source read there; nil for one read whole.
|
||||||
|
read map[string]map[string]bool
|
||||||
|
readAs map[string]catalogue.ArtifactContext
|
||||||
|
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
||||||
|
// closure is not the one the next merge there meets, and the build says no build source.
|
||||||
|
contextsAtHead bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// contextOffHead says a context was not its trunk's head.
|
||||||
|
func (s *sourceInputs) contextOffHead() {
|
||||||
|
if s != nil {
|
||||||
|
s.contextsAtHead = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newSourceInputs(module string) *sourceInputs {
|
func newSourceInputs(module string) *sourceInputs {
|
||||||
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}}
|
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{},
|
||||||
|
own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
|
||||||
|
contextsAtHead: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownHas adds entries, relative to the module's directory, to its build source in its own repository.
|
||||||
|
func (s *sourceInputs) ownHas(entries ...string) {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
s.own[e] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownWhole says an artifact's build source in the module's own repository is not known: the module's
|
||||||
|
// whole directory is its source, as it was before.
|
||||||
|
func (s *sourceInputs) ownWhole() {
|
||||||
|
if s != nil {
|
||||||
|
s.ownIsAll = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contextKey(c catalogue.ArtifactContext) string { return c.Repository + "#" + c.Ref }
|
||||||
|
|
||||||
|
// readIn adds entries to the build source read in a context; one read whole stays whole.
|
||||||
|
func (s *sourceInputs) readIn(c catalogue.ArtifactContext, entries []string) {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := contextKey(c)
|
||||||
|
s.readAs[key] = c
|
||||||
|
set, known := s.read[key]
|
||||||
|
if known && set == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if set == nil {
|
||||||
|
set = map[string]bool{}
|
||||||
|
s.read[key] = set
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
set[e] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readWhole says a context is read whole by an artifact.
|
||||||
|
func (s *sourceInputs) readWhole(c catalogue.ArtifactContext) {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := contextKey(c)
|
||||||
|
s.readAs[key] = c
|
||||||
|
s.read[key] = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSources is what the build says it was made from, per repository: its own (module.json always,
|
||||||
|
// and every artifact's) unless an artifact's is not known, and each context not read whole.
|
||||||
|
func (s *sourceInputs) buildSources() []BuildSource {
|
||||||
|
if s == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []BuildSource
|
||||||
|
if !s.ownIsAll {
|
||||||
|
own := []string{withPrefix(s.prefix, ManifestName)}
|
||||||
|
for e := range s.own {
|
||||||
|
own = append(own, withPrefix(s.prefix, e))
|
||||||
|
}
|
||||||
|
sort.Strings(own)
|
||||||
|
out = append(out, BuildSource{Paths: compactSorted(own)})
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for k := range s.read {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
set := s.read[k]
|
||||||
|
if set == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for e := range set {
|
||||||
|
paths = append(paths, e)
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
c := s.readAs[k]
|
||||||
|
out = append(out, BuildSource{Repository: c.Repository, Ref: c.Ref, Paths: paths})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withPrefix is an entry relative to the module's directory made relative to its repository's root.
|
||||||
|
func withPrefix(prefix, entry string) string {
|
||||||
|
switch {
|
||||||
|
case entry == "." || entry == "./":
|
||||||
|
entry = ""
|
||||||
|
case entry == "**" || entry == "./**" || entry == "/**":
|
||||||
|
if prefix == "" {
|
||||||
|
return "**"
|
||||||
|
}
|
||||||
|
return prefix + "/**"
|
||||||
|
}
|
||||||
|
entry = strings.TrimPrefix(entry, "./")
|
||||||
|
if prefix == "" {
|
||||||
|
if entry == "" {
|
||||||
|
return "./"
|
||||||
|
}
|
||||||
|
return entry
|
||||||
|
}
|
||||||
|
if entry == "" {
|
||||||
|
return prefix + "/"
|
||||||
|
}
|
||||||
|
return prefix + "/" + entry
|
||||||
|
}
|
||||||
|
|
||||||
|
func compactSorted(in []string) []string {
|
||||||
|
var out []string
|
||||||
|
for i, e := range in {
|
||||||
|
if i == 0 || e != in[i-1] {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// notPinned marks the build as one its source does not pin; the first reason stands.
|
// notPinned marks the build as one its source does not pin; the first reason stands.
|
||||||
|
|||||||
@@ -322,6 +322,17 @@ func (b *Build) problems(module string) []string {
|
|||||||
"everything else brings its own recipe", module, a.Name, a.Kind))
|
"everything else brings its own recipe", module, a.Name, a.Kind))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if a.Compiles != "" {
|
||||||
|
if a.Kind != ArtifactImage {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+
|
||||||
|
"told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind))
|
||||||
|
} else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." ||
|
||||||
|
strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles))
|
||||||
|
}
|
||||||
|
}
|
||||||
// An upstream image is named, not read from the repository, so the path rule does not
|
// An upstream image is named, not read from the repository, so the path rule does not
|
||||||
// apply to it — and applying it anyway would refuse every reference with a registry host
|
// apply to it — and applying it anyway would refuse every reference with a registry host
|
||||||
// in it.
|
// in it.
|
||||||
|
|||||||
@@ -180,3 +180,24 @@ func TestAResourceNamingAPackageIsRefused(t *testing.T) {
|
|||||||
t.Fatal("a resource backed by a package was accepted; a package is not a resource")
|
t.Fatal("a resource backed by a package was accepted; a package is not a resource")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An image names the Go package its recipe compiles (novox/hq ADR 0267): within the tree it is built in,
|
||||||
|
// and only an image says one.
|
||||||
|
func TestOnlyAnImageNamesThePackageItCompilesWithinItsTree(t *testing.T) {
|
||||||
|
m, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||||
|
{"name":"x","kind":"image","from":"Dockerfile","compiles":"cmd/x"}]}}`))
|
||||||
|
if err != nil || m.Build.Artifacts[0].Compiles != "cmd/x" {
|
||||||
|
t.Fatalf("an image naming its package was refused or lost it: %v", err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct{ artifact, says string }{
|
||||||
|
{`{"name":"x","kind":"archive","from":"files","compiles":"cmd/x"}`, "Only an image"},
|
||||||
|
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"../x"}`, "leaves the tree"},
|
||||||
|
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"/x"}`, "leaves the tree"},
|
||||||
|
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"a/../../x"}`, "leaves the tree"},
|
||||||
|
} {
|
||||||
|
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[` + c.artifact + `]}}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("%s: %v, wanted a refusal saying %q", c.artifact, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -905,6 +905,16 @@ type Artifact struct {
|
|||||||
// image built from this same module's own repository, the same as every other artifact.
|
// image built from this same module's own repository, the same as every other artifact.
|
||||||
Context *ArtifactContext `json:"context,omitempty"`
|
Context *ArtifactContext `json:"context,omitempty"`
|
||||||
|
|
||||||
|
// Compiles is the Go package an image's recipe compiles, relative to the tree it is built in — its
|
||||||
|
// context, or the module's directory (novox/hq ADR 0267 rule 1).
|
||||||
|
//
|
||||||
|
// **What a build is made from is derived, never listed.** The build seat reads the package's import
|
||||||
|
// closure and hands the recipe that and nothing else, so a merge elsewhere in a shared repository is
|
||||||
|
// no change to this image, and a recipe copying a file outside the closure fails by name instead of
|
||||||
|
// building from something no merge is mapped onto. Empty for an image that compiles no Go: its whole
|
||||||
|
// tree is its build source, as before.
|
||||||
|
Compiles string `json:"compiles,omitempty"`
|
||||||
|
|
||||||
// System is the operating system this artifact is compiled for, for a bundle whose output is a
|
// System is the operating system this artifact is compiled for, for a bundle whose output is a
|
||||||
// binary rather than portable code (novox/hq ADR 0142).
|
// binary rather than portable code (novox/hq ADR 0142).
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -224,6 +224,12 @@ type BuildResult struct {
|
|||||||
// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
|
// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
|
||||||
Read []ReadRepository `json:"read,omitempty"`
|
Read []ReadRepository `json:"read,omitempty"`
|
||||||
|
|
||||||
|
// Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): the
|
||||||
|
// entries the planner maps the next merge's changed files onto. Said only for a build of a commit on
|
||||||
|
// the trunk, and only for a repository whose every artifact's build source the builder knows; a
|
||||||
|
// repository it says nothing of is read whole, as before. Empty from a builder that predates it.
|
||||||
|
Sources []BuildSource `json:"sources,omitempty"`
|
||||||
|
|
||||||
// Trunk is the repository's default branch at the build, and OnTrunk whether the commit built is on it
|
// Trunk is the repository's default branch at the build, and OnTrunk whether the commit built is on it
|
||||||
// (novox/hq ADR 0238): **only a commit on the trunk is published** — the controller refuses to register
|
// (novox/hq ADR 0238): **only a commit on the trunk is published** — the controller refuses to register
|
||||||
// a build of one off it. Empty Trunk is a build seat that could not say, or predates the rule.
|
// a build of one off it. Empty Trunk is a build seat that could not say, or predates the rule.
|
||||||
@@ -263,6 +269,16 @@ type ReadRepository struct {
|
|||||||
Ref string `json:"ref,omitempty"`
|
Ref string `json:"ref,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref
|
||||||
|
// a context's, as its manifest names it, and empty for the module's own; Paths the entries, relative to
|
||||||
|
// that repository's root — `dir/` a Go package's directory but its tests, `dir/**` everything below a
|
||||||
|
// directory, `**` the whole tree, anything else one file (builder.SourceHolds).
|
||||||
|
type BuildSource struct {
|
||||||
|
Repository string `json:"repository,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Paths []string `json:"paths"`
|
||||||
|
}
|
||||||
|
|
||||||
// MadeArtifact is one thing a build produced, as a person would want it reported.
|
// MadeArtifact is one thing a build produced, as a person would want it reported.
|
||||||
type MadeArtifact struct {
|
type MadeArtifact struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
|
|||||||
Reference in New Issue
Block a user