Judge an upstream pin's host like any name, and harden the upstream unpack (review 35, hq issue 345)

The url was skipped by the installation check, so an installation's host or
an address passed; it is judged whole now, the vendor's host declared in the
artifact's names-on-purpose, and an address or a reserved name as the host
is refused. Only the version's dotted-quad reading is skipped. The unpack
caps its entries, drains xz before waiting for it, reads a pax global
header and a nested within's parents, and a redirect off https is not
followed.

cmd/mesh-builder/Dockerfile

@@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
 FROM ${ALPINE_BASE}
 # git to clone what it is asked to build, and the docker client to build and push it. The daemon
 # is the machine's, reached through its socket — a build machine shares the runtime it was given
-# rather than running one inside itself.
+# rather than running one inside itself. xz to read an upstream release an archive is pinned to
-RUN apk add --no-cache git docker-cli
+# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
+RUN apk add --no-cache git docker-cli xz
 COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
 ENTRYPOINT ["/usr/local/bin/mesh-builder"]

internal/builder/builder.go

@@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
 		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
 
 	case catalogue.ArtifactArchive:
-		src.ownHas(cleanEntry(a.From) + "/**")
+		var body []byte
-		body, err := pack(filepath.Join(tree, a.From))
+		var err error
+		if a.URL != "" {
+			// An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which
+			// is in module.json, always part of it.
+			body, err = upstreamArchive(ctx, workspace, a, say)
+			if err != nil {
+				return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+			}
+		} else {
+			src.ownHas(cleanEntry(a.From) + "/**")
+			body, err = pack(filepath.Join(tree, a.From))
+		}
 		if err != nil {
 			return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
 		}

internal/builder/upstream_file.go

@@ -0,0 +1,326 @@
+package builder
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
+// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi.
+//
+// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any
+// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest,
+// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not
+// the pin's fails the build and names both digests, because the address serving something else is the one
+// thing the pin exists to catch.
+//
+// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only
+// its newest version at one address (Owl) would otherwise break every later build of the same pin.
+
+// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine
+// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
+const maxUpstream = 512 << 20
+
+// maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build
+// seat's inodes and the node's. Betterbird's release holds 64.
+const maxEntries = 100_000
+
+// noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says
+// to whoever is on the path what the build seat fetches.
+func noDowngrade(req *http.Request, via []*http.Request) error {
+	if req.URL.Scheme != "https" {
+		return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
+	}
+	if len(via) >= 10 {
+		return errors.New("more than 10 redirects")
+	}
+	return nil
+}
+
+// upstreamClient fetches upstream files: a variable so a test serves its own.
+var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
+
+// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256.
+func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") }
+
+// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an
+// earlier build) and checked, then laid out as the artifact says and packed.
+func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) ([]byte, error) {
+	file, err := upstreamFile(ctx, workspace, a, say)
+	if err != nil {
+		return nil, err
+	}
+	laid := filepath.Join(workspace, "upstream-"+a.Name)
+	if err := os.RemoveAll(laid); err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(laid)
+	if err := os.MkdirAll(laid, 0o755); err != nil {
+		return nil, err
+	}
+	if a.File != "" {
+		body, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil {
+			return nil, err
+		}
+	} else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil {
+		return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err)
+	}
+	return pack(laid)
+}
+
+// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes
+// are still the pin's, else fetched now and kept only when they are.
+func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) (string, error) {
+	want := strings.ToLower(a.SHA256)
+	dir := upstreamDir(workspace)
+	kept := filepath.Join(dir, want)
+	if got, err := sumOf(kept); err == nil {
+		if got == want {
+			say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want))
+			return kept, nil
+		}
+		// Changed on disk since it was kept: fetched again rather than trusted.
+		os.Remove(kept)
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	say("upstream", "fetching %s (%s)", a.URL, a.Version)
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
+	if err != nil {
+		return "", err
+	}
+	client := *upstreamClient
+	client.CheckRedirect = noDowngrade
+	response, err := client.Do(request)
+	if err != nil {
+		return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("%s answered %s", a.URL, response.Status)
+	}
+	part, err := os.CreateTemp(dir, want+".*.part")
+	if err != nil {
+		return "", err
+	}
+	defer os.Remove(part.Name())
+	hash := sha256.New()
+	n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
+	if cerr := part.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", a.URL, err)
+	}
+	if n > maxUpstream {
+		return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream)
+	}
+	got := hex.EncodeToString(hash.Sum(nil))
+	if got != want {
+		return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+
+			"was kept. The address serves another file than the one reviewed: compare it, then move the pin "+
+			"in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n)
+	}
+	if err := os.Rename(part.Name(), kept); err != nil {
+		return "", err
+	}
+	say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want))
+	return kept, nil
+}
+
+// sumOf is the sha256 of a file, in hexadecimal.
+func sumOf(file string) (string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	hash := sha256.New()
+	if _, err := io.Copy(hash, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a
+// directory: of its entries only those within `within` (all, when empty), with that prefix taken off.
+//
+// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside
+// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and
+// directories, and an archive quietly unpacked without what it held would be a release that is not the one
+// pinned.
+func unpackUpstream(ctx context.Context, file, into, within string) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	buffered := bufio.NewReader(f)
+	magic, _ := buffered.Peek(6)
+	var stream io.Reader
+	var wait func() error
+	switch {
+	case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
+		zipped, err := gzip.NewReader(buffered)
+		if err != nil {
+			return err
+		}
+		defer zipped.Close()
+		stream = zipped
+	case bytes.HasPrefix(magic, []byte("BZh")):
+		stream = bzip2.NewReader(buffered)
+	case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
+		// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
+		if _, err := exec.LookPath("xz"); err != nil {
+			return errors.New("it is compressed with xz, and this build seat has no xz to read it")
+		}
+		cmd := exec.CommandContext(ctx, "xz", "-dc")
+		cmd.Stdin = buffered
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+		out, err := cmd.StdoutPipe()
+		if err != nil {
+			return err
+		}
+		if err := cmd.Start(); err != nil {
+			return err
+		}
+		stream = out
+		wait = func() error {
+			if err := cmd.Wait(); err != nil {
+				return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
+			}
+			return nil
+		}
+		defer func() {
+			if wait != nil {
+				_ = cmd.Process.Kill()
+				_ = cmd.Wait()
+			}
+		}()
+	default:
+		return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
+			"whole by naming it in `file`")
+	}
+	if err := untar(stream, into, within); err != nil {
+		return err
+	}
+	if wait != nil {
+		// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
+		if _, err := io.Copy(io.Discard, stream); err != nil {
+			return err
+		}
+		w := wait
+		wait = nil
+		return w()
+	}
+	return nil
+}
+
+// untar writes a tar stream's files and directories within `within` into a directory.
+func untar(stream io.Reader, into, within string) error {
+	reader := tar.NewReader(stream)
+	prefix := ""
+	if within != "" {
+		prefix = path.Clean(within) + "/"
+	}
+	var total int64
+	files, entries := 0, 0
+	for {
+		header, err := reader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if entries++; entries > maxEntries {
+			return fmt.Errorf("it holds more than %d entries", maxEntries)
+		}
+		if header.Typeflag == tar.TypeXGlobalHeader {
+			// A pax global header (git archive writes one) names no file.
+			continue
+		}
+		name := path.Clean(strings.TrimPrefix(header.Name, "./"))
+		if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
+			return fmt.Errorf("%s names a path outside the archive", header.Name)
+		}
+		if name == "." || name+"/" == prefix ||
+			(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
+			// The archive's root, the directory taken, or one of its parents.
+			continue
+		}
+		if prefix != "" {
+			if !strings.HasPrefix(name, prefix) {
+				return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
+			}
+			name = strings.TrimPrefix(name, prefix)
+		}
+		target := filepath.Join(into, filepath.FromSlash(name))
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			total += header.Size
+			if total > maxUpstream {
+				return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
+			}
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return err
+			}
+			mode := os.FileMode(0o644)
+			if header.Mode&0o111 != 0 {
+				mode = 0o755
+			}
+			out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
+				out.Close()
+				return err
+			}
+			if err := out.Close(); err != nil {
+				return err
+			}
+			files++
+		default:
+			return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name,
+				header.Typeflag)
+		}
+	}
+	if files == 0 {
+		if within != "" {
+			return fmt.Errorf("it holds no file within %s", within)
+		}
+		return errors.New("it holds no file")
+	}
+	return nil
+}

internal/builder/upstream_file_test.go

@@ -0,0 +1,385 @@
+package builder
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
+// whose download does not match its sha256 fails the build and names both digests").
+
+type tarEntry struct {
+	name, body string
+	mode       int64
+	kind       byte
+}
+
+func tarOf(t *testing.T, entries []tarEntry) []byte {
+	t.Helper()
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for _, e := range entries {
+		kind := e.kind
+		if kind == 0 {
+			kind = tar.TypeReg
+		}
+		h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
+		if kind == tar.TypeSymlink {
+			h.Linkname, h.Size = "/usr/share/hunspell", 0
+		}
+		if kind == tar.TypeDir {
+			h.Size = 0
+		}
+		if err := w.WriteHeader(h); err != nil {
+			t.Fatal(err)
+		}
+		if kind == tar.TypeReg {
+			if _, err := w.Write([]byte(e.body)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return raw.Bytes()
+}
+
+func gzipped(t *testing.T, body []byte) []byte {
+	t.Helper()
+	var out bytes.Buffer
+	z := gzip.NewWriter(&out)
+	if _, err := z.Write(body); err != nil {
+		t.Fatal(err)
+	}
+	if err := z.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return out.Bytes()
+}
+
+func sumHex(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+// vendor serves files by path over https and counts what it was asked.
+type vendor struct {
+	files map[string][]byte
+	asked int
+}
+
+func serveVendor(t *testing.T, v *vendor) string {
+	t.Helper()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		v.asked++
+		body, ok := v.files[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Write(body)
+	}))
+	t.Cleanup(server.Close)
+	before := upstreamClient
+	// Under a public name, as a manifest must say it: example.com, which the test server's certificate
+	// names, dialled at the server.
+	transport := server.Client().Transport.(*http.Transport).Clone()
+	addr := server.Listener.Addr().String()
+	transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, network, addr)
+	}
+	upstreamClient = &http.Client{Transport: transport}
+	t.Cleanup(func() { upstreamClient = before })
+	return "https://example.com"
+}
+
+// entriesOf lists a packed archive: each file with its mode and content.
+func entriesOf(t *testing.T, packed []byte) map[string]string {
+	t.Helper()
+	z, err := gzip.NewReader(bytes.NewReader(packed))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := tar.NewReader(z)
+	out := map[string]string{}
+	for {
+		h, err := r.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, _ := io.ReadAll(r)
+		out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
+	}
+}
+
+func release(t *testing.T) []byte {
+	return gzipped(t, tarOf(t, []tarEntry{
+		{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
+		{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
+		{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
+		{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
+	}))
+}
+
+func pinnedManifest(base, sum string) string {
+	return `{"module":"betterbird","version":"1",
+  "build":{"artifacts":[
+    {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
+  "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
+}
+
+func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	digest, _ := got.Manifest.Resources[0]["digest"].(string)
+	if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
+		t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
+			got.Manifest.Resources[0], r.archives)
+	}
+
+	packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
+		func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if "sha256:"+sumHex(packed) != digest {
+		t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
+	}
+	entries := entriesOf(t, packed)
+	want := map[string]string{
+		"application.ini":                "[App]\nVersion=153.4.0\n -rw-r--r--",
+		"betterbird":                     "#!/bin/sh\n -rwxr-xr-x",
+		"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
+	}
+	if len(entries) != len(want) {
+		t.Fatalf("the archive holds %v, want %v", entries, want)
+	}
+	for name, w := range want {
+		if entries[name] != w {
+			t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
+		}
+	}
+}
+
+func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
+	// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
+	// must still build after the address moved on.
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/release.tar.gz"] = []byte("the vendor's next release")
+	second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
+	}
+	if v.asked != 1 {
+		t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
+	}
+	if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
+		t.Fatal("two builds of one pin gave two digests")
+	}
+}
+
+func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
+	pinned := release(t)
+	served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
+	_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err == nil {
+		t.Fatal("a download that is not the pinned file was built")
+	}
+	for _, digest := range []string{sumHex(pinned), sumHex(served)} {
+		if !strings.Contains(err.Error(), "sha256:"+digest) {
+			t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
+		}
+	}
+	if len(r.archives) != 0 {
+		t.Fatalf("something was published: %v", r.archives)
+	}
+	kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
+	if len(kept) != 0 {
+		t.Fatalf("the refused download was kept: %v", kept)
+	}
+}
+
+func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
+	xpi := []byte("PK\x03\x04 an add-on")
+	v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
+		URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries := entriesOf(t, packed)
+	if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
+		t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
+	}
+}
+
+func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
+	cases := map[string][]tarEntry{
+		// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
+		"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
+		"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "README", body: "x", mode: 0o644}},
+		"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
+		"no file within":      {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
+	}
+	for want, entries := range cases {
+		body := gzipped(t, tarOf(t, entries))
+		v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+		base := serveVendor(t, v)
+		_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+			URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
+		}
+	}
+}
+
+func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
+	// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
+	}
+	plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(plain)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	for n := range entriesOf(t, packed) {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	if strings.Join(names, ",") != "betterbird" {
+		t.Fatalf("the xz release unpacked to %v", names)
+	}
+}
+
+func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
+	// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
+	// followed by bytes after the tar's end, which must be read before xz is waited for.
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
+	w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
+	w.Write([]byte("x"))
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
+		t.Fatalf("%v", e)
+	}
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz")
+	}
+	// A megabyte after the tar's end: more than a pipe holds.
+	trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(trailing)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/r.tar.xz"] = compressed
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
+		SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for i := 0; i <= maxEntries; i++ {
+		w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
+	}
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
+	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
+	t.Cleanup(plain.Close)
+	secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
+	}))
+	t.Cleanup(secure.Close)
+	before := upstreamClient
+	upstreamClient = secure.Client()
+	t.Cleanup(func() { upstreamClient = before })
+	_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
+		SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "not https") {
+		t.Fatalf("%v", err)
+	}
+}

internal/catalogue/build.go

@@ -2,7 +2,10 @@ package catalogue
 
 import (
 	"fmt"
+	"net"
+	"net/url"
 	"path"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -312,7 +315,7 @@ func (b *Build) problems(module string) []string {
 					module, a.Name, a.System, spokenSystems()))
 			}
 		} else {
-			if a.From == "" {
+			if a.From == "" && a.URL == "" {
 				problems = append(problems, fmt.Sprintf(
 					"%s: %q says nothing about what it is built from", module, a.Name))
 			}
@@ -350,6 +353,85 @@ func (b *Build) problems(module string) []string {
 			problems = append(problems, fmt.Sprintf(
 				"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
 		}
+		problems = append(problems, upstreamFileProblems(module, a)...)
+	}
+	return problems
+}
+
+// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
+var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
+	".test", ".invalid", ".example", ".intranet", ".private"}
+
+// notAPublicHost says why a host is not a public vendor's name, or "".
+func notAPublicHost(host string) string {
+	h := strings.ToLower(strings.TrimSuffix(host, "."))
+	if net.ParseIP(h) != nil {
+		return "an address rather than a name"
+	}
+	if !strings.Contains(h, ".") {
+		return "a name with no domain"
+	}
+	for _, s := range reservedSuffixes {
+		if strings.HasSuffix(h, s) {
+			return "a name under " + s + ", which is nobody's public domain"
+		}
+	}
+	return ""
+}
+
+// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
+var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
+// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
+// besides; a file name or a directory within is a plain path that stays where it is put.
+func upstreamFileProblems(module string, a Artifact) []string {
+	pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
+	if !pinned {
+		return nil
+	}
+	var problems []string
+	say := func(format string, args ...any) {
+		problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
+	}
+	if a.Kind != ArtifactArchive {
+		say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
+			"mirrored from one (novox/hq ADR 0316)", a.Kind)
+		return problems
+	}
+	if a.URL == "" {
+		say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
+		return problems
+	}
+	if a.From != "" {
+		say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
+	}
+	if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
+		say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
+	} else if why := notAPublicHost(u.Hostname()); why != "" {
+		say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
+	}
+	if !sha256Hex.MatchString(a.SHA256) {
+		say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
+			"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
+	}
+	if strings.TrimSpace(a.Version) == "" {
+		say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
+	}
+	if a.File != "" && a.Within != "" {
+		say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
+			"of an archive", a.File, a.Within)
+	}
+	for field, p := range map[string]string{"file": a.File, "within": a.Within} {
+		if p == "" {
+			continue
+		}
+		clean := path.Clean(p)
+		if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
+			(field == "file" && strings.Contains(p, "/")) {
+			say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
+				"file": "file name", "within": "directory"}[field])
+		}
 	}
 	return problems
 }

internal/catalogue/installation.go

@@ -71,6 +71,9 @@ var hostname = regexp.MustCompile(
 		`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
 		`internal|example|tld|test|invalid)`)
 
+// upstreamVersion is where an artifact pinned by URL says which release it is (Artifact.Version).
+var upstreamVersion = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.version$`)
+
 // address is a dotted quad.
 var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
 
@@ -120,7 +123,10 @@ func InstallationProblems(m Manifest) []string {
 			problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at))
 		}
 		for _, ip := range addressesIn(value) {
-			if meant[ip] {
+			// An upstream pin's version may have four dotted numbers (Owl 1.5.6.1), which is no address; a name
+			// in it is still judged above. Its URL is judged whole: the vendor's host goes through the artifact's
+			// names-on-purpose like any name a definition means (novox/hq ADR 0316).
+			if meant[ip] || upstreamVersion.MatchString(at) {
 				continue
 			}
 			problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at))

internal/catalogue/manifest.go

@@ -976,6 +976,35 @@ type Artifact struct {
 	// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
 	Binary string `json:"binary,omitempty"`
 
+	// URL is where an archive's upstream file is fetched from, for an archive that is a release
+	// somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6).
+	// With it come SHA256 and Version, and From stays empty.
+	//
+	// **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of
+	// about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an
+	// add-on whose licence forbids copying it to third parties may not be in a repository at all. So the
+	// manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and
+	// names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own
+	// archive in the artifact store, fetched by digest like every other, and a machine never asks the
+	// vendor. Upgrading is changing the pin, reviewed like any other change.
+	URL string `json:"url,omitempty"`
+	// SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters.
+	SHA256 string `json:"sha256,omitempty"`
+	// Version is what the vendor calls the release pinned, said where a person reads the pin; the build
+	// does not read it.
+	Version string `json:"version,omitempty"`
+	// File is the name an upstream file that is not an archive is kept under: the archive built from it
+	// holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz)
+	// whose files are the artifact.
+	File string `json:"file,omitempty"`
+	// Within is the directory inside an upstream tar archive whose content is the artifact, for a
+	// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
+	// the build. Empty means the archive's root.
+	Within string `json:"within,omitempty"`
+	// NamesOnPurpose are the names this artifact means to name, each with why: the vendor's host an upstream
+	// file is fetched from, which the installation check judges like any name (InstallationProblems).
+	NamesOnPurpose map[string]string `json:"names-on-purpose,omitempty"`
+
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.
 	//

internal/catalogue/upstream_file_test.go

@@ -0,0 +1,101 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build
+// seat, never a copy in the repository.
+
+const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2"
+
+func upstreamManifest(artifact string) string {
+	return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}`
+}
+
+func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
+	for _, a := range []string{
+		`{"name":"archive","kind":"archive","url":"https://vendor.example.org/release-1.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1.0","within":"release"}`,
+		`{"name":"addon","kind":"archive","url":"https://vendor.example.org/addon.xpi","sha256":"` + pinnedSum +
+			`","version":"1.5.6.1","file":"addon.xpi"}`,
+	} {
+		if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
+			t.Errorf("%s was refused: %v", a, err)
+		}
+	}
+}
+
+func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
+	cases := map[string]string{
+		// No digest: what the address serves changes under the mesh.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`:                "without a pinned sha256",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256",
+		// No version: a person reading the pin cannot tell which release.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version",
+		// Not https.
+		`{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https",
+		// Both a directory of the repository and a URL.
+		`{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "one or the other",
+		// A pin without a URL.
+		`{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url",
+		// Only an archive is mirrored from a file.
+		`{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "only an archive",
+		// A file name that is a path, a directory within that leaves the archive, and both at once.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum +
+			`","version":"1","file":"../a.xpi"}`: "not a plain relative file name",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"../up"}`: "not a plain relative directory",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole",
+	}
+	for artifact, want := range cases {
+		_, err := ParseManifest([]byte(upstreamManifest(artifact)))
+		if err == nil {
+			t.Errorf("%s was accepted", artifact)
+			continue
+		}
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err)
+		}
+	}
+}
+
+func TestAnUpstreamFilesVendorIsNamedOnPurposeAndAnInstallationsIsRefused(t *testing.T) {
+	// The vendor's host is judged like any name, and passes through names-on-purpose; a version of four dotted
+	// numbers is not an address, and a name in it is still judged.
+	m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
+		pinnedSum + `","version":"1.5.6.1","file":"owl.xpi","names-on-purpose":{"vendor.be":"Owl's vendor publishes it there"}}`)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if problems := InstallationProblems(m); len(problems) != 0 {
+		t.Fatalf("the declared vendor and the version were taken for an installation: %v", problems)
+	}
+	m.Build.Artifacts[0].NamesOnPurpose = nil
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "vendor.be") {
+		t.Fatalf("an undeclared vendor host is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].Version = "git.novox.be 10.0.0.1"
+	if problems := InstallationProblems(m); len(problems) != 2 {
+		t.Fatalf("a name in the version is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].URL, m.Build.Artifacts[0].Version = "https://mail.novox.be/owl.xpi", "1"
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
+		t.Fatalf("an installation's host in the url is judged: %v", problems)
+	}
+}
+
+func TestAnUpstreamFileFromAnAddressOrAReservedNameIsRefused(t *testing.T) {
+	for _, u := range []string{"https://git.novox.internal/r.tar.xz", "https://10.1.2.3/r.tar.xz",
+		"https://203.0.113.9/r.tar.xz", "https://[2001:db8::1]/r.tar.xz", "https://nas.lan/r.tar.xz", "https://vendor/r.tar.xz"} {
+		_, err := ParseManifest([]byte(upstreamManifest(`{"name":"x","kind":"archive","url":"` + u + `","sha256":"` +
+			pinnedSum + `","version":"1"}`)))
+		if err == nil || !strings.Contains(err.Error(), "published under a public name") {
+			t.Errorf("%s: %v", u, err)
+		}
+	}
+}