Judge an upstream pin's host like any name, and harden the upstream unpack (review 35, hq issue 345)

The url was skipped by the installation check, so an installation's host or
an address passed; it is judged whole now, the vendor's host declared in the
artifact's names-on-purpose, and an address or a reserved name as the host
is refused. Only the version's dotted-quad reading is skipped. The unpack
caps its entries, drains xz before waiting for it, reads a pax global
header and a nested within's parents, and a redirect off https is not
followed.

internal/builder/upstream_file.go

@@ -39,6 +39,22 @@ import (
 // unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
 const maxUpstream = 512 << 20
 
+// maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build
+// seat's inodes and the node's. Betterbird's release holds 64.
+const maxEntries = 100_000
+
+// noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says
+// to whoever is on the path what the build seat fetches.
+func noDowngrade(req *http.Request, via []*http.Request) error {
+	if req.URL.Scheme != "https" {
+		return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
+	}
+	if len(via) >= 10 {
+		return errors.New("more than 10 redirects")
+	}
+	return nil
+}
+
 // upstreamClient fetches upstream files: a variable so a test serves its own.
 var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
 
@@ -98,7 +114,9 @@ func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
 	if err != nil {
 		return "", err
 	}
-	response, err := upstreamClient.Do(request)
+	client := *upstreamClient
+	client.CheckRedirect = noDowngrade
+	response, err := client.Do(request)
 	if err != nil {
 		return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
 	}
@@ -213,6 +231,10 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
 		return err
 	}
 	if wait != nil {
+		// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
+		if _, err := io.Copy(io.Discard, stream); err != nil {
+			return err
+		}
 		w := wait
 		wait = nil
 		return w()
@@ -228,7 +250,7 @@ func untar(stream io.Reader, into, within string) error {
 		prefix = path.Clean(within) + "/"
 	}
 	var total int64
-	files := 0
+	files, entries := 0, 0
 	for {
 		header, err := reader.Next()
 		if err == io.EOF {
@@ -237,11 +259,20 @@ func untar(stream io.Reader, into, within string) error {
 		if err != nil {
 			return err
 		}
+		if entries++; entries > maxEntries {
+			return fmt.Errorf("it holds more than %d entries", maxEntries)
+		}
+		if header.Typeflag == tar.TypeXGlobalHeader {
+			// A pax global header (git archive writes one) names no file.
+			continue
+		}
 		name := path.Clean(strings.TrimPrefix(header.Name, "./"))
 		if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
 			return fmt.Errorf("%s names a path outside the archive", header.Name)
 		}
-		if name == "." || name+"/" == prefix {
+		if name == "." || name+"/" == prefix ||
+			(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
+			// The archive's root, the directory taken, or one of its parents.
 			continue
 		}
 		if prefix != "" {

internal/builder/upstream_file_test.go

@@ -7,7 +7,9 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -16,6 +18,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
 )
@@ -97,9 +100,16 @@ func serveVendor(t *testing.T, v *vendor) string {
 	}))
 	t.Cleanup(server.Close)
 	before := upstreamClient
-	upstreamClient = server.Client()
+	// Under a public name, as a manifest must say it: example.com, which the test server's certificate
+	// names, dialled at the server.
+	transport := server.Client().Transport.(*http.Transport).Clone()
+	addr := server.Listener.Addr().String()
+	transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, network, addr)
+	}
+	upstreamClient = &http.Client{Transport: transport}
 	t.Cleanup(func() { upstreamClient = before })
-	return server.URL
+	return "https://example.com"
 }
 
 // entriesOf lists a packed archive: each file with its mode and content.
@@ -297,3 +307,79 @@ func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
 		t.Fatalf("the xz release unpacked to %v", names)
 	}
 }
+
+func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
+	// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
+	// followed by bytes after the tar's end, which must be read before xz is waited for.
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
+	w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
+	w.Write([]byte("x"))
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
+		t.Fatalf("%v", e)
+	}
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz")
+	}
+	// A megabyte after the tar's end: more than a pipe holds.
+	trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(trailing)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/r.tar.xz"] = compressed
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
+		SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for i := 0; i <= maxEntries; i++ {
+		w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
+	}
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
+	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
+	t.Cleanup(plain.Close)
+	secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
+	}))
+	t.Cleanup(secure.Close)
+	before := upstreamClient
+	upstreamClient = secure.Client()
+	t.Cleanup(func() { upstreamClient = before })
+	_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
+		SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "not https") {
+		t.Fatalf("%v", err)
+	}
+}