From 9d108feab8e70b2f88e63b73fbdcad82f924ca7b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 12:57:30 +0200 Subject: [PATCH] Let an archive be an upstream release pinned by URL and sha256, mirrored by the build seat (hq issue 345, ADR 0316) A mail client's release is about 80 MB per version, too large to keep in a repository at every upgrade, and an add-on whose licence forbids copying it to third parties may not be in a repository at all. An archive artifact may now name url, sha256 and version (and file or within) instead of from: the build seat fetches it once, refuses it when its digest is not the pin's and names both, keeps it in its workspace under the digest (a vendor that serves only its newest version cannot break a later build of the same pin), and packs it like any archive, so a node gets the mesh's own archive by digest and never asks the vendor. The build seat's image carries xz, which Go's standard library does not read. The installation check takes the vendor's address and version of such a pin for the world's, as a registry is an image's. --- cmd/mesh-builder/Dockerfile | 5 +- internal/builder/builder.go | 15 +- internal/builder/upstream_file.go | 295 ++++++++++++++++++++++ internal/builder/upstream_file_test.go | 299 +++++++++++++++++++++++ internal/catalogue/build.go | 60 ++++- internal/catalogue/installation.go | 9 + internal/catalogue/manifest.go | 26 ++ internal/catalogue/upstream_file_test.go | 82 +++++++ 8 files changed, 786 insertions(+), 5 deletions(-) create mode 100644 internal/builder/upstream_file.go create mode 100644 internal/builder/upstream_file_test.go create mode 100644 internal/catalogue/upstream_file_test.go diff --git a/cmd/mesh-builder/Dockerfile b/cmd/mesh-builder/Dockerfile index 3951536d..57e93564 100644 --- a/cmd/mesh-builder/Dockerfile +++ b/cmd/mesh-builder/Dockerfile @@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes FROM ${ALPINE_BASE} # git to clone what it is asked to build, and the docker client to build and push it. The daemon # is the machine's, reached through its socket — a build machine shares the runtime it was given -# rather than running one inside itself. -RUN apk add --no-cache git docker-cli +# rather than running one inside itself. xz to read an upstream release an archive is pinned to +# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz. +RUN apk add --no-cache git docker-cli xz COPY --from=build /mesh-builder /usr/local/bin/mesh-builder ENTRYPOINT ["/usr/local/bin/mesh-builder"] diff --git a/internal/builder/builder.go b/internal/builder/builder.go index e142ef29..5ce8ae9c 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactArchive: - src.ownHas(cleanEntry(a.From) + "/**") - body, err := pack(filepath.Join(tree, a.From)) + var body []byte + var err error + if a.URL != "" { + // An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which + // is in module.json, always part of it. + body, err = upstreamArchive(ctx, workspace, a, say) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) + } + } else { + src.ownHas(cleanEntry(a.From) + "/**") + body, err = pack(filepath.Join(tree, a.From)) + } if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) } diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go new file mode 100644 index 00000000..d5ef52ee --- /dev/null +++ b/internal/builder/upstream_file.go @@ -0,0 +1,295 @@ +package builder + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316 +// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi. +// +// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any +// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest, +// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not +// the pin's fails the build and names both digests, because the address serving something else is the one +// thing the pin exists to catch. +// +// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only +// its newest version at one address (Owl) would otherwise break every later build of the same pin. + +// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine +// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive). +const maxUpstream = 512 << 20 + +// upstreamClient fetches upstream files: a variable so a test serves its own. +var upstreamClient = &http.Client{Timeout: 15 * time.Minute} + +// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256. +func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") } + +// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an +// earlier build) and checked, then laid out as the artifact says and packed. +func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact, + say func(step, format string, args ...any)) ([]byte, error) { + file, err := upstreamFile(ctx, workspace, a, say) + if err != nil { + return nil, err + } + laid := filepath.Join(workspace, "upstream-"+a.Name) + if err := os.RemoveAll(laid); err != nil { + return nil, err + } + defer os.RemoveAll(laid) + if err := os.MkdirAll(laid, 0o755); err != nil { + return nil, err + } + if a.File != "" { + body, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil { + return nil, err + } + } else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil { + return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err) + } + return pack(laid) +} + +// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes +// are still the pin's, else fetched now and kept only when they are. +func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact, + say func(step, format string, args ...any)) (string, error) { + want := strings.ToLower(a.SHA256) + dir := upstreamDir(workspace) + kept := filepath.Join(dir, want) + if got, err := sumOf(kept); err == nil { + if got == want { + say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want)) + return kept, nil + } + // Changed on disk since it was kept: fetched again rather than trusted. + os.Remove(kept) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + say("upstream", "fetching %s (%s)", a.URL, a.Version) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil) + if err != nil { + return "", err + } + response, err := upstreamClient.Do(request) + if err != nil { + return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s answered %s", a.URL, response.Status) + } + part, err := os.CreateTemp(dir, want+".*.part") + if err != nil { + return "", err + } + defer os.Remove(part.Name()) + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1)) + if cerr := part.Close(); err == nil { + err = cerr + } + if err != nil { + return "", fmt.Errorf("fetching %s: %w", a.URL, err) + } + if n > maxUpstream { + return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream) + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != want { + return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+ + "was kept. The address serves another file than the one reviewed: compare it, then move the pin "+ + "in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n) + } + if err := os.Rename(part.Name(), kept); err != nil { + return "", err + } + say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want)) + return kept, nil +} + +// sumOf is the sha256 of a file, in hexadecimal. +func sumOf(file string) (string, error) { + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a +// directory: of its entries only those within `within` (all, when empty), with that prefix taken off. +// +// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside +// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and +// directories, and an archive quietly unpacked without what it held would be a release that is not the one +// pinned. +func unpackUpstream(ctx context.Context, file, into, within string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + buffered := bufio.NewReader(f) + magic, _ := buffered.Peek(6) + var stream io.Reader + var wait func() error + switch { + case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}): + zipped, err := gzip.NewReader(buffered) + if err != nil { + return err + } + defer zipped.Close() + stream = zipped + case bytes.HasPrefix(magic, []byte("BZh")): + stream = bzip2.NewReader(buffered) + case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): + // Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile). + if _, err := exec.LookPath("xz"); err != nil { + return errors.New("it is compressed with xz, and this build seat has no xz to read it") + } + cmd := exec.CommandContext(ctx, "xz", "-dc") + cmd.Stdin = buffered + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + stream = out + wait = func() error { + if err := cmd.Wait(); err != nil { + return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String())) + } + return nil + } + defer func() { + if wait != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + default: + return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " + + "whole by naming it in `file`") + } + if err := untar(stream, into, within); err != nil { + return err + } + if wait != nil { + w := wait + wait = nil + return w() + } + return nil +} + +// untar writes a tar stream's files and directories within `within` into a directory. +func untar(stream io.Reader, into, within string) error { + reader := tar.NewReader(stream) + prefix := "" + if within != "" { + prefix = path.Clean(within) + "/" + } + var total int64 + files := 0 + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := path.Clean(strings.TrimPrefix(header.Name, "./")) + if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") { + return fmt.Errorf("%s names a path outside the archive", header.Name) + } + if name == "." || name+"/" == prefix { + continue + } + if prefix != "" { + if !strings.HasPrefix(name, prefix) { + return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within) + } + name = strings.TrimPrefix(name, prefix) + } + target := filepath.Join(into, filepath.FromSlash(name)) + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + total += header.Size + if total > maxUpstream { + return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + mode := os.FileMode(0o644) + if header.Mode&0o111 != 0 { + mode = 0o755 + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil { + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + files++ + default: + return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name, + header.Typeflag) + } + } + if files == 0 { + if within != "" { + return fmt.Errorf("it holds no file within %s", within) + } + return errors.New("it holds no file") + } + return nil +} diff --git a/internal/builder/upstream_file_test.go b/internal/builder/upstream_file_test.go new file mode 100644 index 00000000..6e233dce --- /dev/null +++ b/internal/builder/upstream_file_test.go @@ -0,0 +1,299 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact +// whose download does not match its sha256 fails the build and names both digests"). + +type tarEntry struct { + name, body string + mode int64 + kind byte +} + +func tarOf(t *testing.T, entries []tarEntry) []byte { + t.Helper() + var raw bytes.Buffer + w := tar.NewWriter(&raw) + for _, e := range entries { + kind := e.kind + if kind == 0 { + kind = tar.TypeReg + } + h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))} + if kind == tar.TypeSymlink { + h.Linkname, h.Size = "/usr/share/hunspell", 0 + } + if kind == tar.TypeDir { + h.Size = 0 + } + if err := w.WriteHeader(h); err != nil { + t.Fatal(err) + } + if kind == tar.TypeReg { + if _, err := w.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return raw.Bytes() +} + +func gzipped(t *testing.T, body []byte) []byte { + t.Helper() + var out bytes.Buffer + z := gzip.NewWriter(&out) + if _, err := z.Write(body); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func sumHex(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// vendor serves files by path over https and counts what it was asked. +type vendor struct { + files map[string][]byte + asked int +} + +func serveVendor(t *testing.T, v *vendor) string { + t.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v.asked++ + body, ok := v.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Write(body) + })) + t.Cleanup(server.Close) + before := upstreamClient + upstreamClient = server.Client() + t.Cleanup(func() { upstreamClient = before }) + return server.URL +} + +// entriesOf lists a packed archive: each file with its mode and content. +func entriesOf(t *testing.T, packed []byte) map[string]string { + t.Helper() + z, err := gzip.NewReader(bytes.NewReader(packed)) + if err != nil { + t.Fatal(err) + } + r := tar.NewReader(z) + out := map[string]string{} + for { + h, err := r.Next() + if err == io.EOF { + return out + } + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(r) + out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String() + } +} + +func release(t *testing.T) []byte { + return gzipped(t, tarOf(t, []tarEntry{ + {name: "betterbird/", kind: tar.TypeDir, mode: 0o755}, + {name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644}, + {name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}, + {name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644}, + })) +} + +func pinnedManifest(base, sum string) string { + return `{"module":"betterbird","version":"1", + "build":{"artifacts":[ + {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]}, + "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}` +} + +func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { + t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", + got.Manifest.Resources[0], r.archives) + } + + packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, + func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if "sha256:"+sumHex(packed) != digest { + t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) + } + entries := entriesOf(t, packed) + want := map[string]string{ + "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", + "betterbird": "#!/bin/sh\n -rwxr-xr-x", + "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", + } + if len(entries) != len(want) { + t.Fatalf("the archive holds %v, want %v", entries, want) + } + for name, w := range want { + if entries[name] != w { + t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) + } + } +} + +func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { + // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once + // must still build after the address moved on. + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + v.files["/release.tar.gz"] = []byte("the vendor's next release") + second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) + } + if v.asked != 1 { + t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) + } + if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { + t.Fatal("two builds of one pin gave two digests") + } +} + +func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { + pinned := release(t) + served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) + v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a download that is not the pinned file was built") + } + for _, digest := range []string{sumHex(pinned), sumHex(served)} { + if !strings.Contains(err.Error(), "sha256:"+digest) { + t.Errorf("the refusal does not name sha256:%s: %v", digest, err) + } + } + if len(r.archives) != 0 { + t.Fatalf("something was published: %v", r.archives) + } + kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) + if len(kept) != 0 { + t.Fatalf("the refused download was kept: %v", kept) + } +} + +func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) { + xpi := []byte("PK\x03\x04 an add-on") + v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive", + URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + entries := entriesOf(t, packed) + if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" { + t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries) + } +} + +func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) { + cases := map[string][]tarEntry{ + // The community package links the dictionaries; a release carrying a link is not one a machine unpacks. + "is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755}, + {name: "betterbird/dictionaries", kind: tar.TypeSymlink}}, + "outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755}, + {name: "README", body: "x", mode: 0o644}}, + "outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}}, + "no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}}, + } + for want, entries := range cases { + body := gzipped(t, tarOf(t, entries)) + v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} + base := serveVendor(t, v) + _, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err) + } + } +} + +func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { + // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") + } + plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(plain) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + var names []string + for n := range entriesOf(t, packed) { + names = append(names, n) + } + sort.Strings(names) + if strings.Join(names, ",") != "betterbird" { + t.Fatalf("the xz release unpacked to %v", names) + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 21f0e296..74815e39 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -2,7 +2,9 @@ package catalogue import ( "fmt" + "net/url" "path" + "regexp" "sort" "strings" ) @@ -312,7 +314,7 @@ func (b *Build) problems(module string) []string { module, a.Name, a.System, spokenSystems())) } } else { - if a.From == "" { + if a.From == "" && a.URL == "" { problems = append(problems, fmt.Sprintf( "%s: %q says nothing about what it is built from", module, a.Name)) } @@ -350,6 +352,62 @@ func (b *Build) problems(module string) []string { problems = append(problems, fmt.Sprintf( "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) } + problems = append(problems, upstreamFileProblems(module, a)...) + } + return problems +} + +// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters. +var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an +// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository +// besides; a file name or a directory within is a plain path that stays where it is put. +func upstreamFileProblems(module string, a Artifact) []string { + pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != "" + if !pinned { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...)) + } + if a.Kind != ArtifactArchive { + say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+ + "mirrored from one (novox/hq ADR 0316)", a.Kind) + return problems + } + if a.URL == "" { + say("names a sha256, a version, a file or a directory within, and no url to fetch it from") + return problems + } + if a.From != "" { + say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL) + } + if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" { + say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL) + } + if !sha256Hex.MatchString(a.SHA256) { + say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+ + "address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL) + } + if strings.TrimSpace(a.Version) == "" { + say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL) + } + if a.File != "" && a.Within != "" { + say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+ + "of an archive", a.File, a.Within) + } + for field, p := range map[string]string{"file": a.File, "within": a.Within} { + if p == "" { + continue + } + clean := path.Clean(p) + if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || + (field == "file" && strings.Contains(p, "/")) { + say("names %s %q, which is not a plain relative %s", field, p, map[string]string{ + "file": "file name", "within": "directory"}[field]) + } } return problems } diff --git a/internal/catalogue/installation.go b/internal/catalogue/installation.go index d1325e72..94733e05 100644 --- a/internal/catalogue/installation.go +++ b/internal/catalogue/installation.go @@ -71,6 +71,10 @@ var hostname = regexp.MustCompile( `(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` + `internal|example|tld|test|invalid)`) +// upstreamPin is where an artifact pinned by URL says where it is published and which release it is: the +// vendor's address and the vendor's version, which name no installation (Artifact.URL). +var upstreamPin = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.(url|version)$`) + // address is a dotted quad. var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`) @@ -113,6 +117,11 @@ func InstallationProblems(m Manifest) []string { "%s is named after %s, and a module is named for what it is, not for where it runs", m.Module, name)) } walk(tree, "", nil, func(at string, value string, meant map[string]bool, isImage bool) { + // An archive pinned by URL names the vendor that publishes it, and the version it pins, which may + // have four dotted numbers (novox/hq ADR 0316): the world's, as a public registry is an image's. + if upstreamPin.MatchString(at) { + return + } for _, name := range namesIn(value) { if (isImage && worldsRegistries[strings.ToLower(name)]) || meant[name] { continue diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index efb32aea..6ba3f51c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -976,6 +976,32 @@ type Artifact struct { // success, and was invisible to the launcher (novox/hq 04-ISSUES/142). Binary string `json:"binary,omitempty"` + // URL is where an archive's upstream file is fetched from, for an archive that is a release + // somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6). + // With it come SHA256 and Version, and From stays empty. + // + // **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of + // about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an + // add-on whose licence forbids copying it to third parties may not be in a repository at all. So the + // manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and + // names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own + // archive in the artifact store, fetched by digest like every other, and a machine never asks the + // vendor. Upgrading is changing the pin, reviewed like any other change. + URL string `json:"url,omitempty"` + // SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters. + SHA256 string `json:"sha256,omitempty"` + // Version is what the vendor calls the release pinned, said where a person reads the pin; the build + // does not read it. + Version string `json:"version,omitempty"` + // File is the name an upstream file that is not an archive is kept under: the archive built from it + // holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz) + // whose files are the artifact. + File string `json:"file,omitempty"` + // Within is the directory inside an upstream tar archive whose content is the artifact, for a + // release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses + // the build. Empty means the archive's root. + Within string `json:"within,omitempty"` + // Entrypoints are the compiled files a tool host should load from this module, relative to the // bundle's root. // diff --git a/internal/catalogue/upstream_file_test.go b/internal/catalogue/upstream_file_test.go new file mode 100644 index 00000000..c4bbcda9 --- /dev/null +++ b/internal/catalogue/upstream_file_test.go @@ -0,0 +1,82 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build +// seat, never a copy in the repository. + +const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2" + +func upstreamManifest(artifact string) string { + return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}` +} + +func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) { + for _, a := range []string{ + `{"name":"archive","kind":"archive","url":"https://vendor.invalid/release-1.tar.xz","sha256":"` + pinnedSum + + `","version":"1.0","within":"release"}`, + `{"name":"addon","kind":"archive","url":"https://vendor.invalid/addon.xpi","sha256":"` + pinnedSum + + `","version":"1.5.6.1","file":"addon.xpi"}`, + } { + if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil { + t.Errorf("%s was refused: %v", a, err) + } + } +} + +func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) { + cases := map[string]string{ + // No digest: what the address serves changes under the mesh. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`: "without a pinned sha256", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256", + // No version: a person reading the pin cannot tell which release. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version", + // Not https. + `{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https", + // Both a directory of the repository and a URL. + `{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1"}`: "one or the other", + // A pin without a URL. + `{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url", + // Only an archive is mirrored from a file. + `{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1"}`: "only an archive", + // A file name that is a path, a directory within that leaves the archive, and both at once. + `{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum + + `","version":"1","file":"../a.xpi"}`: "not a plain relative file name", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1","within":"../up"}`: "not a plain relative directory", + `{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + + `","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole", + } + for artifact, want := range cases { + _, err := ParseManifest([]byte(upstreamManifest(artifact))) + if err == nil { + t.Errorf("%s was accepted", artifact) + continue + } + if !strings.Contains(err.Error(), want) { + t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err) + } + } +} + +func TestAnUpstreamFilesVendorAndVersionNameNoInstallation(t *testing.T) { + // The vendor's address is the world's, as a public registry is an image's; a version of four dotted numbers + // is not an address. Anything else still is judged. + m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` + + pinnedSum + `","version":"1.5.6.1","file":"owl.xpi"}`))) + if err != nil { + t.Fatal(err) + } + if problems := InstallationProblems(m); len(problems) != 0 { + t.Fatalf("the vendor and the version were taken for an installation: %v", problems) + } + m.Build.Artifacts[0].File = "mail.novox.be" + if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") { + t.Fatalf("a name elsewhere in the artifact is still judged: %v", problems) + } +}