307 lines
10 KiB
Go
307 lines
10 KiB
Go
package builder
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
)
|
|
|
|
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
|
|
// whose download does not match its sha256 fails the build and names both digests").
|
|
|
|
type tarEntry struct {
|
|
name, body string
|
|
mode int64
|
|
kind byte
|
|
}
|
|
|
|
func tarOf(t *testing.T, entries []tarEntry) []byte {
|
|
t.Helper()
|
|
var raw bytes.Buffer
|
|
w := tar.NewWriter(&raw)
|
|
for _, e := range entries {
|
|
kind := e.kind
|
|
if kind == 0 {
|
|
kind = tar.TypeReg
|
|
}
|
|
h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
|
|
if kind == tar.TypeSymlink {
|
|
h.Linkname, h.Size = "/usr/share/hunspell", 0
|
|
}
|
|
if kind == tar.TypeDir {
|
|
h.Size = 0
|
|
}
|
|
if err := w.WriteHeader(h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kind == tar.TypeReg {
|
|
if _, err := w.Write([]byte(e.body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw.Bytes()
|
|
}
|
|
|
|
func gzipped(t *testing.T, body []byte) []byte {
|
|
t.Helper()
|
|
var out bytes.Buffer
|
|
z := gzip.NewWriter(&out)
|
|
if _, err := z.Write(body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := z.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Bytes()
|
|
}
|
|
|
|
func sumHex(b []byte) string {
|
|
s := sha256.Sum256(b)
|
|
return hex.EncodeToString(s[:])
|
|
}
|
|
|
|
// vendor serves files by path over https and counts what it was asked.
|
|
type vendor struct {
|
|
files map[string][]byte
|
|
asked int
|
|
}
|
|
|
|
func serveVendor(t *testing.T, v *vendor) string {
|
|
t.Helper()
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
v.asked++
|
|
body, ok := v.files[r.URL.Path]
|
|
if !ok {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Write(body)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
before := upstreamClient
|
|
// Under a public name, as a manifest must say it: example.com, which the test server's certificate
|
|
// names, dialled at the server.
|
|
transport := server.Client().Transport.(*http.Transport).Clone()
|
|
addr := server.Listener.Addr().String()
|
|
transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
|
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
|
}
|
|
upstreamClient = &http.Client{Transport: transport}
|
|
t.Cleanup(func() { upstreamClient = before })
|
|
return "https://example.com"
|
|
}
|
|
|
|
// entriesOf lists a packed archive: each file with its mode and content.
|
|
func entriesOf(t *testing.T, packed []byte) map[string]string {
|
|
t.Helper()
|
|
z, err := gzip.NewReader(bytes.NewReader(packed))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := tar.NewReader(z)
|
|
out := map[string]string{}
|
|
for {
|
|
h, err := r.Next()
|
|
if err == io.EOF {
|
|
return out
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _ := io.ReadAll(r)
|
|
out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
|
|
}
|
|
}
|
|
|
|
func release(t *testing.T) []byte {
|
|
return gzipped(t, tarOf(t, []tarEntry{
|
|
{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
|
|
{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
|
|
{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
|
|
{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
|
|
}))
|
|
}
|
|
|
|
func pinnedManifest(base, sum string) string {
|
|
return `{"module":"betterbird","version":"1",
|
|
"build":{"artifacts":[
|
|
{"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
|
|
"resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
|
|
}
|
|
|
|
func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
|
|
xpi := []byte("PK\x03\x04 an add-on")
|
|
v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
|
|
URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries := entriesOf(t, packed)
|
|
if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
|
|
t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
|
|
cases := map[string][]tarEntry{
|
|
// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
|
|
"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
|
{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
|
|
"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
|
{name: "README", body: "x", mode: 0o644}},
|
|
"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
|
|
"no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
|
|
}
|
|
for want, entries := range cases {
|
|
body := gzipped(t, tarOf(t, entries))
|
|
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), want) {
|
|
t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
|
|
before := maxEntries
|
|
maxEntries = 100
|
|
t.Cleanup(func() { maxEntries = before })
|
|
var raw bytes.Buffer
|
|
w := tar.NewWriter(&raw)
|
|
for i := 0; i <= maxEntries; i++ {
|
|
w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
|
|
}
|
|
w.Close()
|
|
body := gzipped(t, raw.Bytes())
|
|
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
|
base := serveVendor(t, v)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
|
|
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), "entries") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
|
|
plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
|
|
t.Cleanup(plain.Close)
|
|
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
|
|
}))
|
|
t.Cleanup(secure.Close)
|
|
before := upstreamClient
|
|
upstreamClient = secure.Client()
|
|
t.Cleanup(func() { upstreamClient = before })
|
|
_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
|
|
SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
|
|
if err == nil || !strings.Contains(err.Error(), "not https") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
|
|
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
|
|
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
|
|
// read in Go, with nothing on PATH.
|
|
t.Setenv("PATH", "")
|
|
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
|
|
base := serveVendor(t, v)
|
|
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e := entriesOf(t, packed)
|
|
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
|
|
t.Fatalf("%v", e)
|
|
}
|
|
}
|
|
|
|
// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
|
|
// recomputed: a stream asking for a dictionary of its own choosing.
|
|
func withDictionary(t *testing.T, stream []byte, props byte) []byte {
|
|
t.Helper()
|
|
out := append([]byte{}, stream...)
|
|
const at = 12 // the stream header
|
|
size := (int(out[at]) + 1) * 4
|
|
header := out[at : at+size]
|
|
flags := header[1]
|
|
i := 2
|
|
readVarint := func() {
|
|
for header[i]&0x80 != 0 {
|
|
i++
|
|
}
|
|
i++
|
|
}
|
|
if flags&0x40 != 0 {
|
|
readVarint()
|
|
}
|
|
if flags&0x80 != 0 {
|
|
readVarint()
|
|
}
|
|
if header[i] != 0x21 || header[i+1] != 0x01 {
|
|
t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
|
|
}
|
|
header[i+2] = props
|
|
binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
|
|
return out
|
|
}
|
|
|
|
func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
|
|
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
|
|
for _, props := range []byte{36, 40} {
|
|
crafted := withDictionary(t, body, props)
|
|
v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
|
|
base := serveVendor(t, v)
|
|
var before, after runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&before)
|
|
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
|
URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
|
runtime.ReadMemStats(&after)
|
|
if err == nil || !strings.Contains(err.Error(), "dictionary") {
|
|
t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
|
|
}
|
|
if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
|
|
t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
|
|
}
|
|
}
|
|
// The fixture as it is (a small dictionary) still reads.
|
|
if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
|
|
t.Fatal("no fixture")
|
|
}
|
|
}
|