mesh-controller/internal/builder/upstream_file_test.go

package builder

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
)

// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
// whose download does not match its sha256 fails the build and names both digests").

type tarEntry struct {
	name, body string
	mode       int64
	kind       byte
}

func tarOf(t *testing.T, entries []tarEntry) []byte {
	t.Helper()
	var raw bytes.Buffer
	w := tar.NewWriter(&raw)
	for _, e := range entries {
		kind := e.kind
		if kind == 0 {
			kind = tar.TypeReg
		}
		h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
		if kind == tar.TypeSymlink {
			h.Linkname, h.Size = "/usr/share/hunspell", 0
		}
		if kind == tar.TypeDir {
			h.Size = 0
		}
		if err := w.WriteHeader(h); err != nil {
			t.Fatal(err)
		}
		if kind == tar.TypeReg {
			if _, err := w.Write([]byte(e.body)); err != nil {
				t.Fatal(err)
			}
		}
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}
	return raw.Bytes()
}

func gzipped(t *testing.T, body []byte) []byte {
	t.Helper()
	var out bytes.Buffer
	z := gzip.NewWriter(&out)
	if _, err := z.Write(body); err != nil {
		t.Fatal(err)
	}
	if err := z.Close(); err != nil {
		t.Fatal(err)
	}
	return out.Bytes()
}

func sumHex(b []byte) string {
	s := sha256.Sum256(b)
	return hex.EncodeToString(s[:])
}

// vendor serves files by path over https and counts what it was asked.
type vendor struct {
	files map[string][]byte
	asked int
}

func serveVendor(t *testing.T, v *vendor) string {
	t.Helper()
	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		v.asked++
		body, ok := v.files[r.URL.Path]
		if !ok {
			http.NotFound(w, r)
			return
		}
		w.Write(body)
	}))
	t.Cleanup(server.Close)
	before := upstreamClient
	// Under a public name, as a manifest must say it: example.com, which the test server's certificate
	// names, dialled at the server.
	transport := server.Client().Transport.(*http.Transport).Clone()
	addr := server.Listener.Addr().String()
	transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
		return (&net.Dialer{}).DialContext(ctx, network, addr)
	}
	upstreamClient = &http.Client{Transport: transport}
	t.Cleanup(func() { upstreamClient = before })
	return "https://example.com"
}

// entriesOf lists a packed archive: each file with its mode and content.
func entriesOf(t *testing.T, packed []byte) map[string]string {
	t.Helper()
	z, err := gzip.NewReader(bytes.NewReader(packed))
	if err != nil {
		t.Fatal(err)
	}
	r := tar.NewReader(z)
	out := map[string]string{}
	for {
		h, err := r.Next()
		if err == io.EOF {
			return out
		}
		if err != nil {
			t.Fatal(err)
		}
		body, _ := io.ReadAll(r)
		out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
	}
}

func release(t *testing.T) []byte {
	return gzipped(t, tarOf(t, []tarEntry{
		{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
		{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
		{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
		{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
	}))
}

func pinnedManifest(base, sum string) string {
	return `{"module":"betterbird","version":"1",
  "build":{"artifacts":[
    {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
  "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
}

func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
	body := release(t)
	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
	base := serveVendor(t, v)
	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
	got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
		Npmrc{}, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	digest, _ := got.Manifest.Resources[0]["digest"].(string)
	if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
		t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
			got.Manifest.Resources[0], r.archives)
	}

	packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
		URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
		func(string, string, ...any) {})
	if err != nil {
		t.Fatal(err)
	}
	if "sha256:"+sumHex(packed) != digest {
		t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
	}
	entries := entriesOf(t, packed)
	want := map[string]string{
		"application.ini":                "[App]\nVersion=153.4.0\n -rw-r--r--",
		"betterbird":                     "#!/bin/sh\n -rwxr-xr-x",
		"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
	}
	if len(entries) != len(want) {
		t.Fatalf("the archive holds %v, want %v", entries, want)
	}
	for name, w := range want {
		if entries[name] != w {
			t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
		}
	}
}

func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
	// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
	// must still build after the address moved on.
	body := release(t)
	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
	base := serveVendor(t, v)
	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
	first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
		Npmrc{}, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	v.files["/release.tar.gz"] = []byte("the vendor's next release")
	second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
		Npmrc{}, GitCredential{}, nil)
	if err != nil {
		t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
	}
	if v.asked != 1 {
		t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
	}
	if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
		t.Fatal("two builds of one pin gave two digests")
	}
}

func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
	pinned := release(t)
	served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
	v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
	base := serveVendor(t, v)
	r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
	_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
		Npmrc{}, GitCredential{}, nil)
	if err == nil {
		t.Fatal("a download that is not the pinned file was built")
	}
	for _, digest := range []string{sumHex(pinned), sumHex(served)} {
		if !strings.Contains(err.Error(), "sha256:"+digest) {
			t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
		}
	}
	if len(r.archives) != 0 {
		t.Fatalf("something was published: %v", r.archives)
	}
	kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
	if len(kept) != 0 {
		t.Fatalf("the refused download was kept: %v", kept)
	}
}

func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
	xpi := []byte("PK\x03\x04 an add-on")
	v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
	base := serveVendor(t, v)
	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
		URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
	if err != nil {
		t.Fatal(err)
	}
	entries := entriesOf(t, packed)
	if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
		t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
	}
}

func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
	cases := map[string][]tarEntry{
		// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
		"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
			{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
		"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
			{name: "README", body: "x", mode: 0o644}},
		"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
		"no file within":      {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
	}
	for want, entries := range cases {
		body := gzipped(t, tarOf(t, entries))
		v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
		base := serveVendor(t, v)
		_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
			URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
		if err == nil || !strings.Contains(err.Error(), want) {
			t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
		}
	}
}

func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
	// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
	if _, err := exec.LookPath("xz"); err != nil {
		t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
	}
	plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
	cmd := exec.Command("xz", "-zc")
	cmd.Stdin = bytes.NewReader(plain)
	compressed, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
	base := serveVendor(t, v)
	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
		URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
	if err != nil {
		t.Fatal(err)
	}
	var names []string
	for n := range entriesOf(t, packed) {
		names = append(names, n)
	}
	sort.Strings(names)
	if strings.Join(names, ",") != "betterbird" {
		t.Fatalf("the xz release unpacked to %v", names)
	}
}

func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
	// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
	// followed by bytes after the tar's end, which must be read before xz is waited for.
	var raw bytes.Buffer
	w := tar.NewWriter(&raw)
	w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
	w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
	w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
	w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
	w.Write([]byte("x"))
	w.Close()
	body := gzipped(t, raw.Bytes())
	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
	base := serveVendor(t, v)
	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
	if err != nil {
		t.Fatal(err)
	}
	if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
		t.Fatalf("%v", e)
	}
	if _, err := exec.LookPath("xz"); err != nil {
		t.Skip("NOT TESTED here: no xz")
	}
	// A megabyte after the tar's end: more than a pipe holds.
	trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
	cmd := exec.Command("xz", "-zc")
	cmd.Stdin = bytes.NewReader(trailing)
	compressed, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	v.files["/r.tar.xz"] = compressed
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
		SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
		t.Fatal(err)
	}
}

func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
	var raw bytes.Buffer
	w := tar.NewWriter(&raw)
	for i := 0; i <= maxEntries; i++ {
		w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
	}
	w.Close()
	body := gzipped(t, raw.Bytes())
	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
	base := serveVendor(t, v)
	_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
	if err == nil || !strings.Contains(err.Error(), "entries") {
		t.Fatalf("%v", err)
	}
}

func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
	t.Cleanup(plain.Close)
	secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
	}))
	t.Cleanup(secure.Close)
	before := upstreamClient
	upstreamClient = secure.Client()
	t.Cleanup(func() { upstreamClient = before })
	_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
		SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
	if err == nil || !strings.Contains(err.Error(), "not https") {
		t.Fatalf("%v", err)
	}
}