jochen
01fc9e1d73
ulikunitz/xz's ReaderConfig.DictCap is a minimum: a stream asking for a 1 GiB or 4 GiB dictionary was given it. github.com/therootcompany/xz, a port of xz-embedded, takes a maximum and refuses a block whose LZMA2 dictionary is larger before allocating it. 64 MiB, xz's largest preset.
470 lines
17 KiB
Go
470 lines
17 KiB
Go
package builder
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
)
|
|
|
|
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
|
|
// whose download does not match its sha256 fails the build and names both digests").
|
|
|
|
type tarEntry struct {
|
|
name, body string
|
|
mode int64
|
|
kind byte
|
|
}
|
|
|
|
func tarOf(t *testing.T, entries []tarEntry) []byte {
|
|
t.Helper()
|
|
var raw bytes.Buffer
|
|
w := tar.NewWriter(&raw)
|
|
for _, e := range entries {
|
|
kind := e.kind
|
|
if kind == 0 {
|
|
kind = tar.TypeReg
|
|
}
|
|
h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
|
|
if kind == tar.TypeSymlink {
|
|
h.Linkname, h.Size = "/usr/share/hunspell", 0
|
|
}
|
|
if kind == tar.TypeDir {
|
|
h.Size = 0
|
|
}
|
|
if err := w.WriteHeader(h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kind == tar.TypeReg {
|
|
if _, err := w.Write([]byte(e.body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw.Bytes()
|
|
}
|
|
|
|
func gzipped(t *testing.T, body []byte) []byte {
|
|
t.Helper()
|
|
var out bytes.Buffer
|
|
z := gzip.NewWriter(&out)
|
|
if _, err := z.Write(body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := z.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Bytes()
|
|
}
|
|
|
|
func sumHex(b []byte) string {
|
|
s := sha256.Sum256(b)
|
|
return hex.EncodeToString(s[:])
|
|
}
|
|
|
|
// vendor serves files by path over https and counts what it was asked.
|
|
type vendor struct {
|
|
files map[string][]byte
|
|
asked int
|
|
}
|
|
|
|
func serveVendor(t *testing.T, v *vendor) string {
|
|
t.Helper()
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
v.asked++
|
|
body, ok := v.files[r.URL.Path]
|
|
if !ok {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Write(body)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
before := upstreamClient
|
|
// Under a public name, as a manifest must say it: example.com, which the test server's certificate
|
|
// names, dialled at the server.
|
|
transport := server.Client().Transport.(*http.Transport).Clone()
|
|
addr := server.Listener.Addr().String()
|
|
transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
|
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
|
}
|
|
upstreamClient = &http.Client{Transport: transport}
|
|
t.Cleanup(func() { upstreamClient = before })
|
|
return "https://example.com"
|
|
}
|
|
|
|
// entriesOf lists a packed archive: each file with its mode and content.
|
|
func entriesOf(t *testing.T, packed []byte) map[string]string {
|
|
t.Helper()
|
|
z, err := gzip.NewReader(bytes.NewReader(packed))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := tar.NewReader(z)
|
|
out := map[string]string{}
|
|
for {
|
|
h, err := r.Next()
|
|
if err == io.EOF {
|
|
return out
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _ := io.ReadAll(r)
|
|
out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
|
|
}
|
|
}
|
|
|
|
func release(t *testing.T) []byte {
|
|
return gzipped(t, tarOf(t, []tarEntry{
|
|
{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
|
|
{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
|
|
{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
|
|
{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
|
|
}))
|
|
}
|
|
|
|
func pinnedManifest(base, sum string) string {
|
|
return `{"module":"betterbird","version":"1",
|
|
"build":{"artifacts":[
|
|
{"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
|
|
"resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
|
|
}
|
|
|
|
func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
|
|
body := release(t)
|
|
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
|
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
|
Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digest, _ := got.Manifest.Resources[0]["digest"].(string)
|
|
if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
|
|
t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
|
|
got.Manifest.Resources[0], r.archives)
|
|
}
|
|
|
|
packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
|
|
func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if "sha256:"+sumHex(packed) != digest {
|
|
t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
|
|
}
|
|
entries := entriesOf(t, packed)
|
|
want := map[string]string{
|
|
"application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--",
|
|
"betterbird": "#!/bin/sh\n -rwxr-xr-x",
|
|
"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
|
|
}
|
|
if len(entries) != len(want) {
|
|
t.Fatalf("the archive holds %v, want %v", entries, want)
|
|
}
|
|
for name, w := range want {
|
|
if entries[name] != w {
|
|
t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
|
|
// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
|
|
// must still build after the address moved on.
|
|
body := release(t)
|
|
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
|
first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
|
Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v.files["/release.tar.gz"] = []byte("the vendor's next release")
|
|
second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
|
Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
|
|
}
|
|
if v.asked != 1 {
|
|
t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
|
|
}
|
|
if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
|
|
t.Fatal("two builds of one pin gave two digests")
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
|
|
pinned := release(t)
|
|
served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
|
|
v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
|
|
base := serveVendor(t, v)
|
|
r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
|
|
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
|
Npmrc{}, GitCredential{}, nil)
|
|
if err == nil {
|
|
t.Fatal("a download that is not the pinned file was built")
|
|
}
|
|
for _, digest := range []string{sumHex(pinned), sumHex(served)} {
|
|
if !strings.Contains(err.Error(), "sha256:"+digest) {
|
|
t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
|
|
}
|
|
}
|
|
if len(r.archives) != 0 {
|
|
t.Fatalf("something was published: %v", r.archives)
|
|
}
|
|
kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
|
|
if len(kept) != 0 {
|
|
t.Fatalf("the refused download was kept: %v", kept)
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
|
|
xpi := []byte("PK\x03\x04 an add-on")
|
|
v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
|
|
URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries := entriesOf(t, packed)
|
|
if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
|
|
t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
|
|
cases := map[string][]tarEntry{
|
|
// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
|
|
"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
|
{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
|
|
"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
|
{name: "README", body: "x", mode: 0o644}},
|
|
"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
|
|
"no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
|
|
}
|
|
for want, entries := range cases {
|
|
body := gzipped(t, tarOf(t, entries))
|
|
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), want) {
|
|
t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
|
|
// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
|
|
if _, err := exec.LookPath("xz"); err != nil {
|
|
t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
|
|
}
|
|
plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
|
|
cmd := exec.Command("xz", "-zc")
|
|
cmd.Stdin = bytes.NewReader(plain)
|
|
compressed, err := cmd.Output()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for n := range entriesOf(t, packed) {
|
|
names = append(names, n)
|
|
}
|
|
sort.Strings(names)
|
|
if strings.Join(names, ",") != "betterbird" {
|
|
t.Fatalf("the xz release unpacked to %v", names)
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
|
|
// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
|
|
// followed by bytes after the tar's end, which must be read before xz is waited for.
|
|
var raw bytes.Buffer
|
|
w := tar.NewWriter(&raw)
|
|
w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
|
|
w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
|
|
w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
|
|
w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
|
|
w.Write([]byte("x"))
|
|
w.Close()
|
|
body := gzipped(t, raw.Bytes())
|
|
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
|
|
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
|
|
t.Fatalf("%v", e)
|
|
}
|
|
if _, err := exec.LookPath("xz"); err != nil {
|
|
t.Skip("NOT TESTED here: no xz")
|
|
}
|
|
// A megabyte after the tar's end: more than a pipe holds.
|
|
trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
|
|
cmd := exec.Command("xz", "-zc")
|
|
cmd.Stdin = bytes.NewReader(trailing)
|
|
compressed, err := cmd.Output()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v.files["/r.tar.xz"] = compressed
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
|
|
SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
|
|
var raw bytes.Buffer
|
|
w := tar.NewWriter(&raw)
|
|
for i := 0; i <= maxEntries; i++ {
|
|
w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
|
|
}
|
|
w.Close()
|
|
body := gzipped(t, raw.Bytes())
|
|
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
|
|
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), "entries") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
|
|
plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
|
|
t.Cleanup(plain.Close)
|
|
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
|
|
}))
|
|
t.Cleanup(secure.Close)
|
|
before := upstreamClient
|
|
upstreamClient = secure.Client()
|
|
t.Cleanup(func() { upstreamClient = before })
|
|
_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
|
|
SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), "not https") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
|
|
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
|
|
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
|
|
// read in Go, with nothing on PATH.
|
|
t.Setenv("PATH", "")
|
|
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e := entriesOf(t, packed)
|
|
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
|
|
t.Fatalf("%v", e)
|
|
}
|
|
}
|
|
|
|
// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
|
|
// recomputed: a stream asking for a dictionary of its own choosing.
|
|
func withDictionary(t *testing.T, stream []byte, props byte) []byte {
|
|
t.Helper()
|
|
out := append([]byte{}, stream...)
|
|
const at = 12 // the stream header
|
|
size := (int(out[at]) + 1) * 4
|
|
header := out[at : at+size]
|
|
flags := header[1]
|
|
i := 2
|
|
readVarint := func() {
|
|
for header[i]&0x80 != 0 {
|
|
i++
|
|
}
|
|
i++
|
|
}
|
|
if flags&0x40 != 0 {
|
|
readVarint()
|
|
}
|
|
if flags&0x80 != 0 {
|
|
readVarint()
|
|
}
|
|
if header[i] != 0x21 || header[i+1] != 0x01 {
|
|
t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
|
|
}
|
|
header[i+2] = props
|
|
binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
|
|
return out
|
|
}
|
|
|
|
func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
|
|
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
|
|
for _, props := range []byte{36, 40} {
|
|
crafted := withDictionary(t, body, props)
|
|
v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
|
|
base := serveVendor(t, v)
|
|
var before, after runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&before)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
runtime.ReadMemStats(&after)
|
|
if err == nil || !strings.Contains(err.Error(), "dictionary") {
|
|
t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
|
|
}
|
|
if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
|
|
t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
|
|
}
|
|
}
|
|
// The fixture as it is (a small dictionary) still reads.
|
|
if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
|
|
t.Fatal("no fixture")
|
|
}
|
|
}
|